Examples with MycatPropertyConf io.mycat.memory.unsafe.utils.MycatPropertyConf used on opensource projects

Example 1 with MycatPropertyConf

use of io.mycat.memory.unsafe.utils.MycatPropertyConf in project Mycat-Server by MyCATApache.

the class TaskMemoryManagerSuite method offHeapConfigurationBackwardsCompatibility.

@Test
public void offHeapConfigurationBackwardsCompatibility() {
    final MycatPropertyConf conf = new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "true").set("mycat.memory.offHeap.size", "1000");
    final DataNodeMemoryManager manager = new DataNodeMemoryManager(new TestMemoryManager(conf), 0);
    Assert.assertSame(MemoryMode.OFF_HEAP, manager.tungstenMemoryMode);
}

Example 2 with MycatPropertyConf

use of io.mycat.memory.unsafe.utils.MycatPropertyConf in project Mycat-Server by MyCATApache.

the class UnsafeExternalRowSorterTest method testUnsafeExternalRowSorter.

/**
     * 测试类型 LONG，INT，SHORT,Float，Double，String，Binary
     * 经测试基数排序可以适用上述数据类型，大大提高排序速度

     */
@Test
public void testUnsafeExternalRowSorter() throws NoSuchFieldException, IllegalAccessException, IOException {
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeDiskManager blockManager = myCatMemory.getBlockManager();
    SerializerManager serializerManager = myCatMemory.getSerializerManager();
    MycatPropertyConf conf = myCatMemory.getConf();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
         * 1.schema ,模拟一个field字段值
         *
         */
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
    orderCols[0] = orderCol;
    /**
         * 2 .PrefixComputer
         */
    StructType schema = new StructType(colMetaMap, fieldCount);
    schema.setOrderCols(orderCols);
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
    /**
         * 3 .PrefixComparator 默认是ASC，可以选择DESC
         */
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**使用基数排序？true or false*/
    true);
    UnsafeRow unsafeRow;
    BufferHolder bufferHolder;
    UnsafeRowWriter unsafeRowWriter;
    String line = "testUnsafeRow";
    // List<Float> floats = new ArrayList<Float>();
    List<Long> longs = new ArrayList<Long>();
    final Random rand = new Random(42);
    for (int i = 0; i < TEST_SIZE; i++) {
        unsafeRow = new UnsafeRow(3);
        bufferHolder = new BufferHolder(unsafeRow);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        String key = getRandomString(rand.nextInt(300) + 100);
        //long v = rand.nextLong();
        // longs.add(v);
        unsafeRowWriter.write(0, key.getBytes());
        // unsafeRowWriter.write(0, BytesTools.toBytes(v));
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, ("35" + 1).getBytes());
        unsafeRow.setTotalSize(bufferHolder.totalSize());
        sorter.insertRow(unsafeRow);
    }
    Iterator<UnsafeRow> iter = sorter.sort();
    /*
         float [] com = new float[floats.size()];
        for (int i = 0; i <floats.size() ; i++) {
                com[i] = floats.get(i);
        }
        Arrays.sort(com);


        long[] com = new long[longs.size()];
        for (int i = 0; i < longs.size() ; i++) {
            com[i] = longs.get(i);
        }

        Arrays.sort(com);
        */
    UnsafeRow row = null;
    int indexprint = 0;
    while (iter.hasNext()) {
        row = iter.next();
        // LOGGER.error(indexprint + "    " +  row.getUTF8String(0));
        //Assert.assertEquals(com[indexprint],
        //        BytesTools.toLong(row.getBinary(0)));
        // Double c = Double.parseDouble(String.valueOf(com[indexprint])) ;
        // Double c1 = Double.parseDouble(String.valueOf(BytesTools.toFloat(row.getBinary(0)))) ;
        //  Assert.assertEquals(0,c.compareTo(c1));
        indexprint++;
    }
    Assert.assertEquals(TEST_SIZE, indexprint);
}

Example 3 with MycatPropertyConf

use of io.mycat.memory.unsafe.utils.MycatPropertyConf in project Mycat-Server by MyCATApache.

the class UnsafeInMemorySorterSuite method testSortingOnlyByIntegerPrefix.

@Test
public void testSortingOnlyByIntegerPrefix() throws Exception {
    final String[] dataToSort = new String[] { "Boba", "Pearls", "Tapioca", "Taho", "Condensed Milk", "Jasmine", "Milk Tea", "Lychee", "Mango" };
    final DataNodeMemoryManager memoryManager = new DataNodeMemoryManager(new TestMemoryManager(new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "false")), 0);
    final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager);
    final MemoryBlock dataPage = memoryManager.allocatePage(2048, null);
    final Object baseObject = dataPage.getBaseObject();
    // Write the records into the data page:
    long position = dataPage.getBaseOffset();
    for (String str : dataToSort) {
        final byte[] strBytes = str.getBytes(StandardCharsets.UTF_8);
        Platform.putInt(baseObject, position, strBytes.length);
        position += 4;
        Platform.copyMemory(strBytes, Platform.BYTE_ARRAY_OFFSET, baseObject, position, strBytes.length);
        position += strBytes.length;
    }
    // Since the key fits within the 8-byte prefix, we don't need to do any record comparison, so
    // use a dummy comparator
    final RecordComparator recordComparator = new RecordComparator() {

        @Override
        public int compare(Object leftBaseObject, long leftBaseOffset, Object rightBaseObject, long rightBaseOffset) {
            return 0;
        }
    };
    // Compute key prefixes based on the records' partition ids
    final HashPartitioner hashPartitioner = new HashPartitioner(4);
    // Use integer comparison for comparing prefixes (which are partition ids, in this case)
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager, recordComparator, prefixComparator, dataToSort.length, shouldUseRadixSort(), true);
    // Given a page of records, insert those records into the sorter one-by-one:
    position = dataPage.getBaseOffset();
    System.out.println("(0)address = " + position);
    for (int i = 0; i < dataToSort.length; i++) {
        if (!sorter.hasSpaceForAnotherRecord()) {
            sorter.expandPointerArray(consumer.allocateLongArray(sorter.getMemoryUsage() / 8 * 2));
        }
        // position now points to the start of a record (which holds its length).
        final int recordLength = Platform.getInt(baseObject, position);
        final long address = memoryManager.encodePageNumberAndOffset(dataPage, position);
        final String str = getStringFromDataPage(baseObject, position + 4, recordLength);
        final int partitionId = hashPartitioner.getPartition(str);
        System.out.println("(" + partitionId + "," + str + ")");
        sorter.insertRecord(address, partitionId);
        position += 4 + recordLength;
    }
    final UnsafeSorterIterator iter = sorter.getSortedIterator();
    int iterLength = 0;
    long prevPrefix = -1;
    Arrays.sort(dataToSort);
    while (iter.hasNext()) {
        iter.loadNext();
        final String str = getStringFromDataPage(iter.getBaseObject(), iter.getBaseOffset(), iter.getRecordLength());
        final long keyPrefix = iter.getKeyPrefix();
        assertThat(str, isIn(Arrays.asList(dataToSort)));
        assertThat(keyPrefix, greaterThanOrEqualTo(prevPrefix));
        prevPrefix = keyPrefix;
        iterLength++;
    }
    assertEquals(dataToSort.length, iterLength);
}

Example 4 with MycatPropertyConf

use of io.mycat.memory.unsafe.utils.MycatPropertyConf in project Mycat-Server by MyCATApache.

the class BlockManagerTest method testNewDiskBlockWriter.

@Test
public void testNewDiskBlockWriter() {
    MycatPropertyConf conf = new MycatPropertyConf();
    SerializerManager serializerManager = new SerializerManager();
}

Example 5 with MycatPropertyConf

use of io.mycat.memory.unsafe.utils.MycatPropertyConf in project Mycat-Server by MyCATApache.

the class BlockManagerTest method testNewDiskBlockManager.

@Test
public void testNewDiskBlockManager() throws IOException {
    MycatPropertyConf conf = new MycatPropertyConf();
    SerializerManager serializerManager = new SerializerManager();
    DataNodeDiskManager blockManager = new DataNodeDiskManager(conf, true, serializerManager);
    DataNodeFileManager diskBlockManager = blockManager.diskBlockManager();
    /**
         * 生成一个文本文件
         */
    File file = diskBlockManager.getFile("mycat1");
    FileOutputStream fos = new FileOutputStream(file);
    BufferedOutputStream bos = new BufferedOutputStream(fos);
    bos.write("KOKKKKKK".getBytes());
    bos.flush();
    bos.close();
    fos.close();
    /**
         * 读刚刚写入的文件
         */
    File file1 = diskBlockManager.getFile("mycat1");
    FileInputStream ios = new FileInputStream(file1);
    BufferedInputStream bin = new BufferedInputStream(ios);
    byte[] str = new byte["KOKKKKKK".getBytes().length];
    int size = bin.read(str);
    bin.close();
    ios.close();
    Assert.assertEquals("KOKKKKKK", new String(str));
    File file2 = diskBlockManager.getFile("mycat1");
    DiskRowWriter writer = blockManager.getDiskWriter(null, file2, DummySerializerInstance.INSTANCE, 1024 * 1024);
    byte[] writeBuffer = new byte[4];
    int v = 4;
    writeBuffer[0] = (byte) (v >>> 24);
    writeBuffer[1] = (byte) (v >>> 16);
    writeBuffer[2] = (byte) (v >>> 8);
    writeBuffer[3] = (byte) (v >>> 0);
    writer.write(writeBuffer, 0, 4);
    writer.write("you are ok? 1111111111111".getBytes(), 0, "you are ok? 1111111111111".getBytes().length);
    writer.write("you are ok? 1111111111111".getBytes(), 0, "you are ok? 1111111111111".getBytes().length);
    writer.write("you are ok? 1111111111111".getBytes(), 0, "you are ok? 1111111111111".getBytes().length);
    writer.write("you are ok? 1111111111111".getBytes(), 0, "you are ok? 1111111111111".getBytes().length);
    writer.write("you are ok? 1111111111111".getBytes(), 0, "you are ok? 1111111111111".getBytes().length);
    writer.write("you are ok? 1111111111111".getBytes(), 0, "you are ok? 1111111111111".getBytes().length);
    writer.write("you are ok? 1111111111111".getBytes(), 0, "you are ok? 1111111111111".getBytes().length);
    writer.write("you are ok? 1111111111111".getBytes(), 0, "you are ok? 1111111111111".getBytes().length);
    writer.close();
    try {
        Thread.sleep(100);
    } catch (InterruptedException e) {
        logger.error(e.getMessage());
    }
    assert (file2.length() > 0);
    final BufferedInputStream bs = new BufferedInputStream(new FileInputStream(file2));
    try {
        InputStream in = serializerManager.wrapForCompression(null, bs);
        DataInputStream din = new DataInputStream(in);
        int numRecords = din.readInt();
        Assert.assertEquals(4, numRecords);
        din.close();
        in.close();
        bs.close();
    } catch (IOException e) {
        Closeables.close(bs, /* swallowIOException = */
        true);
        throw e;
    }
}