use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.
the class UnsafeRowGrouper method filterHaving.
private void filterHaving(@Nonnull UnsafeExternalRowSorter sorter) {
if (havingCols.getColMeta() == null || aggregationMap == null) {
return;
}
KVIterator<UnsafeRow, UnsafeRow> it = aggregationMap.iterator();
byte[] right = havingCols.getRight().getBytes(StandardCharsets.UTF_8);
int index = havingCols.getColMeta().getColIndex();
try {
while (it.next()) {
UnsafeRow row = getAllBinaryRow(it.getValue());
switch(havingCols.getOperator()) {
case "=":
if (!eq(row.getBinary(index), right)) {
sorter.insertRow(row);
}
break;
case ">":
if (!gt(row.getBinary(index), right)) {
sorter.insertRow(row);
}
break;
case "<":
if (!lt(row.getBinary(index), right)) {
sorter.insertRow(row);
}
break;
case ">=":
if (!gt(row.getBinary(index), right) && eq(row.getBinary(index), right)) {
sorter.insertRow(row);
}
break;
case "<=":
if (!lt(row.getBinary(index), right) && eq(row.getBinary(index), right)) {
sorter.insertRow(row);
}
break;
case "!=":
if (!neq(row.getBinary(index), right)) {
sorter.insertRow(row);
}
break;
}
}
} catch (IOException e) {
logger.error(e.getMessage());
}
}
use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.
the class UnsafeRowGrouper method initGroupKey.
private void initGroupKey() {
/**
* 构造groupKey
*/
Map<String, ColMeta> groupcolMetaMap = new HashMap<String, ColMeta>(this.groupKeyfieldCount);
groupKey = new UnsafeRow(this.groupKeyfieldCount);
bufferHolder = new BufferHolder(groupKey, 0);
unsafeRowWriter = new UnsafeRowWriter(bufferHolder, this.groupKeyfieldCount);
bufferHolder.reset();
ColMeta curColMeta = null;
for (int i = 0; i < this.groupKeyfieldCount; i++) {
curColMeta = this.columToIndx.get(sortColumnsByIndex[i].toUpperCase());
groupcolMetaMap.put(sortColumnsByIndex[i], curColMeta);
switch(curColMeta.colType) {
case ColMeta.COL_TYPE_BIT:
groupKey.setByte(i, (byte) 0);
break;
case ColMeta.COL_TYPE_INT:
case ColMeta.COL_TYPE_INT24:
case ColMeta.COL_TYPE_LONG:
groupKey.setInt(i, 0);
break;
case ColMeta.COL_TYPE_SHORT:
groupKey.setShort(i, (short) 0);
break;
case ColMeta.COL_TYPE_FLOAT:
groupKey.setFloat(i, 0);
break;
case ColMeta.COL_TYPE_DOUBLE:
groupKey.setDouble(i, 0);
break;
case ColMeta.COL_TYPE_NEWDECIMAL:
// groupKey.setDouble(i, 0);
unsafeRowWriter.write(i, new BigDecimal(0L));
break;
case ColMeta.COL_TYPE_LONGLONG:
groupKey.setLong(i, 0);
break;
default:
unsafeRowWriter.write(i, "init".getBytes());
break;
}
}
groupKey.setTotalSize(bufferHolder.totalSize());
groupKeySchema = new StructType(groupcolMetaMap, this.groupKeyfieldCount);
groupKeySchema.setOrderCols(null);
}
use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.
the class UnsafeRowGrouper method getValue.
/**
* 构造value
*/
private UnsafeRow getValue(UnsafeRow row) throws UnsupportedEncodingException {
UnsafeRow value = new UnsafeRow(this.valuefieldCount);
bufferHolder = new BufferHolder(value, 0);
unsafeRowWriter = new UnsafeRowWriter(bufferHolder, this.valuefieldCount);
bufferHolder.reset();
ColMeta curColMeta = null;
for (Map.Entry<String, ColMeta> fieldEntry : columToIndx.entrySet()) {
curColMeta = fieldEntry.getValue();
if (!row.isNullAt(curColMeta.colIndex)) {
switch(curColMeta.colType) {
case ColMeta.COL_TYPE_BIT:
value.setByte(curColMeta.colIndex, row.getByte(curColMeta.colIndex));
break;
case ColMeta.COL_TYPE_INT:
case ColMeta.COL_TYPE_LONG:
case ColMeta.COL_TYPE_INT24:
value.setInt(curColMeta.colIndex, BytesTools.getInt(row.getBinary(curColMeta.colIndex)));
break;
case ColMeta.COL_TYPE_SHORT:
value.setShort(curColMeta.colIndex, BytesTools.getShort(row.getBinary(curColMeta.colIndex)));
break;
case ColMeta.COL_TYPE_LONGLONG:
value.setLong(curColMeta.colIndex, BytesTools.getLong(row.getBinary(curColMeta.colIndex)));
break;
case ColMeta.COL_TYPE_FLOAT:
value.setFloat(curColMeta.colIndex, BytesTools.getFloat(row.getBinary(curColMeta.colIndex)));
break;
case ColMeta.COL_TYPE_DOUBLE:
value.setDouble(curColMeta.colIndex, BytesTools.getDouble(row.getBinary(curColMeta.colIndex)));
break;
case ColMeta.COL_TYPE_NEWDECIMAL:
// value.setDouble(curColMeta.colIndex, BytesTools.getDouble(row.getBinary(curColMeta.colIndex)));
unsafeRowWriter.write(curColMeta.colIndex, new BigDecimal(new String(row.getBinary(curColMeta.colIndex))));
break;
default:
unsafeRowWriter.write(curColMeta.colIndex, row.getBinary(curColMeta.colIndex));
break;
}
} else {
switch(curColMeta.colType) {
case ColMeta.COL_TYPE_NEWDECIMAL:
BigDecimal nullDecimal = null;
unsafeRowWriter.write(curColMeta.colIndex, nullDecimal);
break;
default:
value.setNullAt(curColMeta.colIndex);
break;
}
}
}
value.setTotalSize(bufferHolder.totalSize());
return value;
}
use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.
the class UnsafeRowGrouper method initEmptyValueKey.
private void initEmptyValueKey() {
/**
* 构造valuerow
*/
emptyAggregationBuffer = new UnsafeRow(this.valuefieldCount);
bufferHolder = new BufferHolder(emptyAggregationBuffer, 0);
unsafeRowWriter = new UnsafeRowWriter(bufferHolder, this.valuefieldCount);
bufferHolder.reset();
ColMeta curColMeta = null;
for (Map.Entry<String, ColMeta> fieldEntry : columToIndx.entrySet()) {
curColMeta = fieldEntry.getValue();
switch(curColMeta.colType) {
case ColMeta.COL_TYPE_BIT:
emptyAggregationBuffer.setByte(curColMeta.colIndex, (byte) 0);
break;
case ColMeta.COL_TYPE_INT:
case ColMeta.COL_TYPE_INT24:
case ColMeta.COL_TYPE_LONG:
emptyAggregationBuffer.setInt(curColMeta.colIndex, 0);
break;
case ColMeta.COL_TYPE_SHORT:
emptyAggregationBuffer.setShort(curColMeta.colIndex, (short) 0);
break;
case ColMeta.COL_TYPE_LONGLONG:
emptyAggregationBuffer.setLong(curColMeta.colIndex, 0);
break;
case ColMeta.COL_TYPE_FLOAT:
emptyAggregationBuffer.setFloat(curColMeta.colIndex, 0);
break;
case ColMeta.COL_TYPE_DOUBLE:
emptyAggregationBuffer.setDouble(curColMeta.colIndex, 0);
break;
case ColMeta.COL_TYPE_NEWDECIMAL:
// emptyAggregationBuffer.setDouble(curColMeta.colIndex, 0);
unsafeRowWriter.write(curColMeta.colIndex, new BigDecimal(0L));
break;
default:
unsafeRowWriter.write(curColMeta.colIndex, "init".getBytes());
break;
}
}
emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize());
aggBufferSchema = new StructType(columToIndx, this.valuefieldCount);
aggBufferSchema.setOrderCols(null);
}
use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.
the class UnsafeExternalRowSorterTest method testUnsafeExternalRowSorter.
/**
* 测试类型 LONG,INT,SHORT,Float,Double,String,Binary
* 经测试基数排序可以适用上述数据类型,大大提高排序速度
*/
@Test
public void testUnsafeExternalRowSorter() throws NoSuchFieldException, IllegalAccessException, IOException {
MyCatMemory myCatMemory = new MyCatMemory();
MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
DataNodeDiskManager blockManager = myCatMemory.getBlockManager();
SerializerManager serializerManager = myCatMemory.getSerializerManager();
MycatPropertyConf conf = myCatMemory.getConf();
DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
/**
* 1.schema ,模拟一个field字段值
*
*/
int fieldCount = 3;
ColMeta colMeta = null;
Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
colMetaMap.put("id", colMeta);
colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
colMetaMap.put("name", colMeta);
colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
colMetaMap.put("age", colMeta);
OrderCol[] orderCols = new OrderCol[1];
OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
orderCols[0] = orderCol;
/**
* 2 .PrefixComputer
*/
StructType schema = new StructType(colMetaMap, fieldCount);
schema.setOrderCols(orderCols);
UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
/**
* 3 .PrefixComparator 默认是ASC,可以选择DESC
*/
final PrefixComparator prefixComparator = PrefixComparators.LONG;
UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**使用基数排序?true or false*/
true);
UnsafeRow unsafeRow;
BufferHolder bufferHolder;
UnsafeRowWriter unsafeRowWriter;
String line = "testUnsafeRow";
// List<Float> floats = new ArrayList<Float>();
List<Long> longs = new ArrayList<Long>();
final Random rand = new Random(42);
for (int i = 0; i < TEST_SIZE; i++) {
unsafeRow = new UnsafeRow(3);
bufferHolder = new BufferHolder(unsafeRow);
unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
bufferHolder.reset();
String key = getRandomString(rand.nextInt(300) + 100);
//long v = rand.nextLong();
// longs.add(v);
unsafeRowWriter.write(0, key.getBytes());
// unsafeRowWriter.write(0, BytesTools.toBytes(v));
unsafeRowWriter.write(1, line.getBytes());
unsafeRowWriter.write(2, ("35" + 1).getBytes());
unsafeRow.setTotalSize(bufferHolder.totalSize());
sorter.insertRow(unsafeRow);
}
Iterator<UnsafeRow> iter = sorter.sort();
/*
float [] com = new float[floats.size()];
for (int i = 0; i <floats.size() ; i++) {
com[i] = floats.get(i);
}
Arrays.sort(com);
long[] com = new long[longs.size()];
for (int i = 0; i < longs.size() ; i++) {
com[i] = longs.get(i);
}
Arrays.sort(com);
*/
UnsafeRow row = null;
int indexprint = 0;
while (iter.hasNext()) {
row = iter.next();
// LOGGER.error(indexprint + " " + row.getUTF8String(0));
//Assert.assertEquals(com[indexprint],
// BytesTools.toLong(row.getBinary(0)));
// Double c = Double.parseDouble(String.valueOf(com[indexprint])) ;
// Double c1 = Double.parseDouble(String.valueOf(BytesTools.toFloat(row.getBinary(0)))) ;
// Assert.assertEquals(0,c.compareTo(c1));
indexprint++;
}
Assert.assertEquals(TEST_SIZE, indexprint);
}
Aggregations