use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.
the class TestSorter method runSorter.
public void runSorter(MyCatMemory myCatMemory, MemoryManager memoryManager, MycatPropertyConf conf) throws NoSuchFieldException, IllegalAccessException, IOException {
DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
/**
* 1.schema ,模拟一个field字段值
*
*/
int fieldCount = 3;
ColMeta colMeta = null;
Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
colMetaMap.put("id", colMeta);
colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
colMetaMap.put("name", colMeta);
colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
colMetaMap.put("age", colMeta);
OrderCol[] orderCols = new OrderCol[1];
OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
orderCols[0] = orderCol;
/**
* 2 .PrefixComputer
*/
StructType schema = new StructType(colMetaMap, fieldCount);
schema.setOrderCols(orderCols);
UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
/**
* 3 .PrefixComparator 默认是ASC,可以选择DESC
*/
final PrefixComparator prefixComparator = PrefixComparators.LONG;
UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**使用基数排序?true or false*/
true);
UnsafeRow unsafeRow;
BufferHolder bufferHolder;
UnsafeRowWriter unsafeRowWriter;
String line = "testUnsafeRow";
final Random rand = new Random(42);
for (int i = 0; i < TEST_SIZE; i++) {
unsafeRow = new UnsafeRow(3);
bufferHolder = new BufferHolder(unsafeRow);
unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
bufferHolder.reset();
String key = getRandomString(rand.nextInt(300) + 100);
unsafeRowWriter.write(0, key.getBytes());
unsafeRowWriter.write(1, line.getBytes());
unsafeRowWriter.write(2, ("35" + 1).getBytes());
unsafeRow.setTotalSize(bufferHolder.totalSize());
sorter.insertRow(unsafeRow);
}
Iterator<UnsafeRow> iter = sorter.sort();
UnsafeRow row = null;
int indexprint = 0;
while (iter.hasNext()) {
row = iter.next();
indexprint++;
}
sorter.cleanupResources();
countDownLatch.countDown();
System.out.println("Thread ID :" + Thread.currentThread().getId() + "Index : " + indexprint);
}
use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.
the class DataNodeMergeManager method onRowMetaData.
public void onRowMetaData(Map<String, ColMeta> columToIndx, int fieldCount) throws IOException {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("field metadata keys:" + columToIndx.keySet());
LOGGER.debug("field metadata values:" + columToIndx.values());
}
OrderCol[] orderCols = null;
StructType schema = null;
UnsafeExternalRowSorter.PrefixComputer prefixComputer = null;
PrefixComparator prefixComparator = null;
DataNodeMemoryManager dataNodeMemoryManager = null;
UnsafeExternalRowSorter sorter = null;
int[] groupColumnIndexs = null;
this.fieldCount = fieldCount;
if (rrs.getGroupByCols() != null) {
groupColumnIndexs = toColumnIndex(rrs.getGroupByCols(), columToIndx);
if (LOGGER.isDebugEnabled()) {
for (int i = 0; i < rrs.getGroupByCols().length; i++) {
LOGGER.debug("groupColumnIndexs:" + rrs.getGroupByCols()[i]);
}
}
}
if (rrs.getHavingCols() != null) {
ColMeta colMeta = columToIndx.get(rrs.getHavingCols().getLeft().toUpperCase());
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("getHavingCols:" + rrs.getHavingCols().toString());
}
if (colMeta != null) {
rrs.getHavingCols().setColMeta(colMeta);
}
}
if (rrs.isHasAggrColumn()) {
List<MergeCol> mergCols = new LinkedList<MergeCol>();
Map<String, Integer> mergeColsMap = rrs.getMergeCols();
if (mergeColsMap != null) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("isHasAggrColumn:" + mergeColsMap.toString());
}
for (Map.Entry<String, Integer> mergEntry : mergeColsMap.entrySet()) {
String colName = mergEntry.getKey().toUpperCase();
int type = mergEntry.getValue();
if (MergeCol.MERGE_AVG == type) {
ColMeta sumColMeta = columToIndx.get(colName + "SUM");
ColMeta countColMeta = columToIndx.get(colName + "COUNT");
if (sumColMeta != null && countColMeta != null) {
ColMeta colMeta = new ColMeta(sumColMeta.colIndex, countColMeta.colIndex, sumColMeta.getColType());
mergCols.add(new MergeCol(colMeta, mergEntry.getValue()));
}
} else {
ColMeta colMeta = columToIndx.get(colName);
mergCols.add(new MergeCol(colMeta, mergEntry.getValue()));
}
}
}
// add no alias merg column
for (Map.Entry<String, ColMeta> fieldEntry : columToIndx.entrySet()) {
String colName = fieldEntry.getKey();
int result = MergeCol.tryParseAggCol(colName);
if (result != MergeCol.MERGE_UNSUPPORT && result != MergeCol.MERGE_NOMERGE) {
mergCols.add(new MergeCol(fieldEntry.getValue(), result));
}
}
/**
* Group操作
*/
unsafeRowGrouper = new UnsafeRowGrouper(columToIndx, rrs.getGroupByCols(), mergCols.toArray(new MergeCol[mergCols.size()]), rrs.getHavingCols());
}
if (rrs.getOrderByCols() != null) {
LinkedHashMap<String, Integer> orders = rrs.getOrderByCols();
orderCols = new OrderCol[orders.size()];
int i = 0;
for (Map.Entry<String, Integer> entry : orders.entrySet()) {
String key = StringUtil.removeBackquote(entry.getKey().toUpperCase());
ColMeta colMeta = columToIndx.get(key);
if (colMeta == null) {
throw new IllegalArgumentException("all columns in order by clause should be in the selected column list!" + entry.getKey());
}
orderCols[i++] = new OrderCol(colMeta, entry.getValue());
}
/**
* 构造全局排序器
*/
schema = new StructType(columToIndx, fieldCount);
schema.setOrderCols(orderCols);
prefixComputer = new RowPrefixComputer(schema);
// if(orderCols.length>0
// && orderCols[0].getOrderType()
// == OrderCol.COL_ORDER_TYPE_ASC){
// prefixComparator = PrefixComparators.LONG;
// }else {
// prefixComparator = PrefixComparators.LONG_DESC;
// }
prefixComparator = getPrefixComparator(orderCols);
dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
/**
* 默认排序,只是将数据连续存储到内存中即可。
*/
globalSorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), false, /**是否使用基数排序*/
true);
}
if (conf.getBoolean("mycat.stream.output.result", false) && globalSorter == null && unsafeRowGrouper == null) {
setStreamOutputResult(true);
} else {
/**
* 1.schema
*/
schema = new StructType(columToIndx, fieldCount);
schema.setOrderCols(orderCols);
/**
* 2 .PrefixComputer
*/
prefixComputer = new RowPrefixComputer(schema);
/**
* 3 .PrefixComparator 默认是ASC,可以选择DESC
*/
prefixComparator = PrefixComparators.LONG;
dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
globalMergeResult = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), false, /**是否使用基数排序*/
false);
}
}
use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.
the class TaskMemoryManagerSuite method offHeapConfigurationBackwardsCompatibility.
@Test
public void offHeapConfigurationBackwardsCompatibility() {
final MycatPropertyConf conf = new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "true").set("mycat.memory.offHeap.size", "1000");
final DataNodeMemoryManager manager = new DataNodeMemoryManager(new TestMemoryManager(conf), 0);
Assert.assertSame(MemoryMode.OFF_HEAP, manager.tungstenMemoryMode);
}
use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.
the class UnsafeExternalRowSorterTest method testUnsafeExternalRowSorter.
/**
* 测试类型 LONG,INT,SHORT,Float,Double,String,Binary
* 经测试基数排序可以适用上述数据类型,大大提高排序速度
*/
@Test
public void testUnsafeExternalRowSorter() throws NoSuchFieldException, IllegalAccessException, IOException {
MyCatMemory myCatMemory = new MyCatMemory();
MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
DataNodeDiskManager blockManager = myCatMemory.getBlockManager();
SerializerManager serializerManager = myCatMemory.getSerializerManager();
MycatPropertyConf conf = myCatMemory.getConf();
DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
/**
* 1.schema ,模拟一个field字段值
*
*/
int fieldCount = 3;
ColMeta colMeta = null;
Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
colMetaMap.put("id", colMeta);
colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
colMetaMap.put("name", colMeta);
colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
colMetaMap.put("age", colMeta);
OrderCol[] orderCols = new OrderCol[1];
OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
orderCols[0] = orderCol;
/**
* 2 .PrefixComputer
*/
StructType schema = new StructType(colMetaMap, fieldCount);
schema.setOrderCols(orderCols);
UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
/**
* 3 .PrefixComparator 默认是ASC,可以选择DESC
*/
final PrefixComparator prefixComparator = PrefixComparators.LONG;
UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**使用基数排序?true or false*/
true);
UnsafeRow unsafeRow;
BufferHolder bufferHolder;
UnsafeRowWriter unsafeRowWriter;
String line = "testUnsafeRow";
// List<Float> floats = new ArrayList<Float>();
List<Long> longs = new ArrayList<Long>();
final Random rand = new Random(42);
for (int i = 0; i < TEST_SIZE; i++) {
unsafeRow = new UnsafeRow(3);
bufferHolder = new BufferHolder(unsafeRow);
unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
bufferHolder.reset();
String key = getRandomString(rand.nextInt(300) + 100);
//long v = rand.nextLong();
// longs.add(v);
unsafeRowWriter.write(0, key.getBytes());
// unsafeRowWriter.write(0, BytesTools.toBytes(v));
unsafeRowWriter.write(1, line.getBytes());
unsafeRowWriter.write(2, ("35" + 1).getBytes());
unsafeRow.setTotalSize(bufferHolder.totalSize());
sorter.insertRow(unsafeRow);
}
Iterator<UnsafeRow> iter = sorter.sort();
/*
float [] com = new float[floats.size()];
for (int i = 0; i <floats.size() ; i++) {
com[i] = floats.get(i);
}
Arrays.sort(com);
long[] com = new long[longs.size()];
for (int i = 0; i < longs.size() ; i++) {
com[i] = longs.get(i);
}
Arrays.sort(com);
*/
UnsafeRow row = null;
int indexprint = 0;
while (iter.hasNext()) {
row = iter.next();
// LOGGER.error(indexprint + " " + row.getUTF8String(0));
//Assert.assertEquals(com[indexprint],
// BytesTools.toLong(row.getBinary(0)));
// Double c = Double.parseDouble(String.valueOf(com[indexprint])) ;
// Double c1 = Double.parseDouble(String.valueOf(BytesTools.toFloat(row.getBinary(0)))) ;
// Assert.assertEquals(0,c.compareTo(c1));
indexprint++;
}
Assert.assertEquals(TEST_SIZE, indexprint);
}
use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.
the class UnsafeInMemorySorterSuite method testSortingOnlyByIntegerPrefix.
@Test
public void testSortingOnlyByIntegerPrefix() throws Exception {
final String[] dataToSort = new String[] { "Boba", "Pearls", "Tapioca", "Taho", "Condensed Milk", "Jasmine", "Milk Tea", "Lychee", "Mango" };
final DataNodeMemoryManager memoryManager = new DataNodeMemoryManager(new TestMemoryManager(new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "false")), 0);
final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager);
final MemoryBlock dataPage = memoryManager.allocatePage(2048, null);
final Object baseObject = dataPage.getBaseObject();
// Write the records into the data page:
long position = dataPage.getBaseOffset();
for (String str : dataToSort) {
final byte[] strBytes = str.getBytes(StandardCharsets.UTF_8);
Platform.putInt(baseObject, position, strBytes.length);
position += 4;
Platform.copyMemory(strBytes, Platform.BYTE_ARRAY_OFFSET, baseObject, position, strBytes.length);
position += strBytes.length;
}
// Since the key fits within the 8-byte prefix, we don't need to do any record comparison, so
// use a dummy comparator
final RecordComparator recordComparator = new RecordComparator() {
@Override
public int compare(Object leftBaseObject, long leftBaseOffset, Object rightBaseObject, long rightBaseOffset) {
return 0;
}
};
// Compute key prefixes based on the records' partition ids
final HashPartitioner hashPartitioner = new HashPartitioner(4);
// Use integer comparison for comparing prefixes (which are partition ids, in this case)
final PrefixComparator prefixComparator = PrefixComparators.LONG;
UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager, recordComparator, prefixComparator, dataToSort.length, shouldUseRadixSort(), true);
// Given a page of records, insert those records into the sorter one-by-one:
position = dataPage.getBaseOffset();
System.out.println("(0)address = " + position);
for (int i = 0; i < dataToSort.length; i++) {
if (!sorter.hasSpaceForAnotherRecord()) {
sorter.expandPointerArray(consumer.allocateLongArray(sorter.getMemoryUsage() / 8 * 2));
}
// position now points to the start of a record (which holds its length).
final int recordLength = Platform.getInt(baseObject, position);
final long address = memoryManager.encodePageNumberAndOffset(dataPage, position);
final String str = getStringFromDataPage(baseObject, position + 4, recordLength);
final int partitionId = hashPartitioner.getPartition(str);
System.out.println("(" + partitionId + "," + str + ")");
sorter.insertRecord(address, partitionId);
position += 4 + recordLength;
}
final UnsafeSorterIterator iter = sorter.getSortedIterator();
int iterLength = 0;
long prevPrefix = -1;
Arrays.sort(dataToSort);
while (iter.hasNext()) {
iter.loadNext();
final String str = getStringFromDataPage(iter.getBaseObject(), iter.getBaseOffset(), iter.getRecordLength());
final long keyPrefix = iter.getKeyPrefix();
assertThat(str, isIn(Arrays.asList(dataToSort)));
assertThat(keyPrefix, greaterThanOrEqualTo(prevPrefix));
prevPrefix = keyPrefix;
iterLength++;
}
assertEquals(dataToSort.length, iterLength);
}
Aggregations