Search in sources :

Example 26 with MapMaker

use of com.google.common.collect.MapMaker in project DirectMemory by raffaeleguidi.

the class PreliminarBenchmarks method pumpWithOneAllocation.

private void pumpWithOneAllocation(int ops, byte[] payload) {
    ConcurrentMap<String, ByteBuffer> test = new MapMaker().concurrencyLevel(4).maximumSize(ops).expireAfterWrite(10, TimeUnit.MINUTES).makeMap();
    logger.info(Ram.inMb(ops * payload.length) + " in " + ops + " slices to store");
    ByteBuffer bulk = ByteBuffer.allocateDirect(ops * payload.length);
    double started = System.currentTimeMillis();
    for (int i = 0; i < ops; i++) {
        bulk.position(i * payload.length);
        final ByteBuffer buf = bulk.duplicate();
        buf.put(payload);
        test.put("test-" + i, buf);
    }
    double finished = System.currentTimeMillis();
    logger.info("done in " + (finished - started) / 1000 + " seconds");
    for (ByteBuffer buf : test.values()) {
        buf.clear();
    }
}
Also used : MapMaker(com.google.common.collect.MapMaker) ByteBuffer(java.nio.ByteBuffer)

Example 27 with MapMaker

use of com.google.common.collect.MapMaker in project otter by alibaba.

the class MysqlDialect method initShardColumns.

private void initShardColumns() {
    // soft引用设置,避免内存爆了
    GenericMapMaker mapMaker = null;
    mapMaker = new MapMaker().softValues().evictionListener(new MapEvictionListener<List<String>, Table>() {

        public void onEviction(List<String> names, Table table) {
            logger.warn("Eviction For Table:" + table);
        }
    });
    this.shardColumns = mapMaker.makeComputingMap(new Function<List<String>, String>() {

        public String apply(List<String> names) {
            Assert.isTrue(names.size() == 2);
            try {
                String result = DdlUtils.getShardKeyByDRDS(jdbcTemplate, names.get(0), names.get(0), names.get(1));
                if (StringUtils.isEmpty(result)) {
                    return "";
                } else {
                    return result;
                }
            } catch (Exception e) {
                throw new NestableRuntimeException("find table [" + names.get(0) + "." + names.get(1) + "] error", e);
            }
        }
    });
}
Also used : Function(com.google.common.base.Function) GenericMapMaker(com.google.common.collect.GenericMapMaker) Table(org.apache.ddlutils.model.Table) NestableRuntimeException(org.apache.commons.lang.exception.NestableRuntimeException) GenericMapMaker(com.google.common.collect.GenericMapMaker) MapMaker(com.google.common.collect.MapMaker) List(java.util.List) MapEvictionListener(com.google.common.collect.MapEvictionListener) NestableRuntimeException(org.apache.commons.lang.exception.NestableRuntimeException)

Example 28 with MapMaker

use of com.google.common.collect.MapMaker in project otter by alibaba.

the class DbLoadMergerTest method testMergeWithUpdateKeyOfIUUUU.

/**
     * 测试在主键发生变化后的merge操作,Insert/Update/Update/Update/Update
     */
@Test
public void testMergeWithUpdateKeyOfIUUUU() {
    Map<RowKey, EventData> mergeMap = new MapMaker().makeMap();
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE, KEY_VALUE_NEW1), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW1, KEY_VALUE_NEW2), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW2, KEY_VALUE_NEW3), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW3, KEY_VALUE_NEW4), mergeMap);
    for (Entry<RowKey, EventData> entry : mergeMap.entrySet()) {
        RowKey key = entry.getKey();
        EventColumn keyColumn = key.getKeys().get(0);
        Assert.assertEquals(KEY_VALUE_NEW4, keyColumn.getColumnValue());
        Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());
        EventData eventData = entry.getValue();
        Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
        Assert.assertEquals(TABLE_NAME, eventData.getTableName());
        Assert.assertEquals(TABLE_ID, eventData.getTableId());
        Assert.assertEquals(EventType.INSERT, eventData.getEventType());
        // 不存在oldKeys
        Assert.assertEquals(eventData.getOldKeys().size(), 0);
    }
}
Also used : RowKey(com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) MapMaker(com.google.common.collect.MapMaker) EventData(com.alibaba.otter.shared.etl.model.EventData) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Example 29 with MapMaker

use of com.google.common.collect.MapMaker in project otter by alibaba.

the class DbLoadMergerTest method testMergeWithUpdateKeyOfUU.

/**
     * 测试在主键发生变化后的merge操作,Update/Update
     */
@Test
public void testMergeWithUpdateKeyOfUU() {
    Map<RowKey, EventData> mergeMap = new MapMaker().makeMap();
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE, KEY_VALUE_NEW1), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW1, KEY_VALUE_NEW2), mergeMap);
    for (Entry<RowKey, EventData> entry : mergeMap.entrySet()) {
        RowKey key = entry.getKey();
        EventColumn keyColumn = key.getKeys().get(0);
        Assert.assertEquals(KEY_VALUE_NEW2, keyColumn.getColumnValue());
        Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());
        EventData eventData = entry.getValue();
        Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
        Assert.assertEquals(TABLE_NAME, eventData.getTableName());
        Assert.assertEquals(TABLE_ID, eventData.getTableId());
        Assert.assertEquals(EventType.UPDATE, eventData.getEventType());
        List<EventColumn> oldKeys = eventData.getOldKeys();
        List<EventColumn> keys = eventData.getKeys();
        Assert.assertNotSame(oldKeys, keys);
    }
}
Also used : RowKey(com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) MapMaker(com.google.common.collect.MapMaker) EventData(com.alibaba.otter.shared.etl.model.EventData) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Example 30 with MapMaker

use of com.google.common.collect.MapMaker in project otter by alibaba.

the class DbLoadMergerTest method testMergeWithSameKeyOfIUDI.

/**
     * 测试insert+update+delete+insert
     */
@Test
public void testMergeWithSameKeyOfIUDI() {
    Map<RowKey, EventData> mergeMap = new MapMaker().makeMap();
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(), mergeMap);
    DbLoadMerger.merge(makeDeleteEventData(), mergeMap);
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    for (Entry<RowKey, EventData> entry : mergeMap.entrySet()) {
        RowKey key = entry.getKey();
        EventColumn keyColumn = key.getKeys().get(0);
        Assert.assertEquals(KEY_VALUE, keyColumn.getColumnValue());
        Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());
        EventData eventData = entry.getValue();
        Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
        Assert.assertEquals(TABLE_NAME, eventData.getTableName());
        Assert.assertEquals(TABLE_ID, eventData.getTableId());
        Assert.assertEquals(EventType.INSERT, eventData.getEventType());
        // 不存在oldKeys
        Assert.assertEquals(eventData.getOldKeys().size(), 0);
        List<EventColumn> columns = eventData.getColumns();
        Assert.assertEquals(2, columns.size());
    }
}
Also used : RowKey(com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) MapMaker(com.google.common.collect.MapMaker) EventData(com.alibaba.otter.shared.etl.model.EventData) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Aggregations

MapMaker (com.google.common.collect.MapMaker)32 EventData (com.alibaba.otter.shared.etl.model.EventData)9 Test (org.junit.Test)9 BaseDbTest (com.alibaba.otter.node.etl.BaseDbTest)8 RowKey (com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey)8 EventColumn (com.alibaba.otter.shared.etl.model.EventColumn)8 Test (org.testng.annotations.Test)8 List (java.util.List)7 DateTime (org.joda.time.DateTime)7 Sequence (io.druid.java.util.common.guava.Sequence)6 TimeseriesQueryQueryToolChest (io.druid.query.timeseries.TimeseriesQueryQueryToolChest)5 TimeseriesResultValue (io.druid.query.timeseries.TimeseriesResultValue)5 Interval (org.joda.time.Interval)5 Function (com.google.common.base.Function)4 Result (io.druid.query.Result)3 ByteBuffer (java.nio.ByteBuffer)3 ConfigException (com.alibaba.otter.shared.common.model.config.ConfigException)2 Channel (com.alibaba.otter.shared.common.model.config.channel.Channel)2 Node (com.alibaba.otter.shared.common.model.config.node.Node)2 ComputeFunction (com.alibaba.otter.shared.common.utils.cache.RefreshMemoryMirror.ComputeFunction)2