Examples with DbMediaSource com.alibaba.otter.shared.common.model.config.data.db.DbMediaSource used on opensource projects

Example 11 with DbMediaSource

use of com.alibaba.otter.shared.common.model.config.data.db.DbMediaSource in project otter by alibaba.

the class DataMediaSourceServiceImpl method modelToDo.

/**
     * 类型：数据库类型 Mysql和Oracle 用于Model对象转化为DO对象
     * 
     * @param dataMediaSource
     * @return DataMediaSourceDO
     */
private DataMediaSourceDO modelToDo(DataMediaSource dataMediaSource) {
    DataMediaSourceDO dataMediaSourceDo = new DataMediaSourceDO();
    try {
        dataMediaSourceDo.setId(dataMediaSource.getId());
        dataMediaSourceDo.setName(dataMediaSource.getName());
        dataMediaSourceDo.setType(dataMediaSource.getType());
        if (dataMediaSource instanceof DbMediaSource) {
            dataMediaSourceDo.setProperties(JsonUtils.marshalToString((DbMediaSource) dataMediaSource));
        } else if (dataMediaSource instanceof MqMediaSource) {
            dataMediaSourceDo.setProperties(JsonUtils.marshalToString((MqMediaSource) dataMediaSource));
        }
        dataMediaSourceDo.setGmtCreate(dataMediaSource.getGmtCreate());
        dataMediaSourceDo.setGmtModified(dataMediaSource.getGmtModified());
    } catch (Exception e) {
        logger.error("ERROR ## change the dataMediaSource Model to Do has an exception");
        throw new ManagerException(e);
    }
    return dataMediaSourceDo;
}

Example 12 with DbMediaSource

use of com.alibaba.otter.shared.common.model.config.data.db.DbMediaSource in project otter by alibaba.

the class DataSourceList method execute.

public void execute(@Param("pageIndex") int pageIndex, @Param("searchKey") String searchKey, Context context) throws Exception {
    @SuppressWarnings("unchecked") Map<String, Object> condition = new HashMap<String, Object>();
    if ("请输入关键字(目前支持DataSource的ID、名字搜索)".equals(searchKey)) {
        searchKey = "";
    }
    condition.put("searchKey", searchKey);
    int count = dataMediaSourceService.getCount(condition);
    Paginator paginator = new Paginator();
    paginator.setItems(count);
    paginator.setPage(pageIndex);
    condition.put("offset", paginator.getOffset());
    condition.put("length", paginator.getLength());
    List<DataMediaSource> dataMediaSources = dataMediaSourceService.listByCondition(condition);
    List<SeniorDataMediaSource> seniorDataMediaSources = new ArrayList<SeniorDataMediaSource>();
    for (DataMediaSource dataMediaSource : dataMediaSources) {
        SeniorDataMediaSource seniorDataMediaSource = new SeniorDataMediaSource();
        seniorDataMediaSource.setEncode(dataMediaSource.getEncode());
        seniorDataMediaSource.setGmtCreate(dataMediaSource.getGmtCreate());
        seniorDataMediaSource.setGmtModified(dataMediaSource.getGmtModified());
        seniorDataMediaSource.setId(dataMediaSource.getId());
        seniorDataMediaSource.setName(dataMediaSource.getName());
        seniorDataMediaSource.setType(dataMediaSource.getType());
        if (dataMediaSource instanceof DbMediaSource) {
            seniorDataMediaSource.setDriver(((DbMediaSource) dataMediaSource).getDriver());
            seniorDataMediaSource.setUrl(((DbMediaSource) dataMediaSource).getUrl());
            seniorDataMediaSource.setUsername(((DbMediaSource) dataMediaSource).getUsername());
        } else if (dataMediaSource instanceof MqMediaSource) {
            seniorDataMediaSource.setUrl(((MqMediaSource) dataMediaSource).getUrl());
            seniorDataMediaSource.setStorePath(((MqMediaSource) dataMediaSource).getStorePath());
        }
        List<DataMedia> dataMedia = dataMediaService.listByDataMediaSourceId(dataMediaSource.getId());
        seniorDataMediaSource.setDataMedias(dataMedia);
        if (dataMedia.size() < 1) {
            seniorDataMediaSource.setUsed(false);
        } else {
            seniorDataMediaSource.setUsed(true);
        }
        seniorDataMediaSources.add(seniorDataMediaSource);
    }
    context.put("sources", seniorDataMediaSources);
    context.put("paginator", paginator);
    context.put("searchKey", searchKey);
}

Example 13 with DbMediaSource

use of com.alibaba.otter.shared.common.model.config.data.db.DbMediaSource in project otter by alibaba.

the class DBDataSourceService method destroy.

public void destroy(Long pipelineId) {
    Map<DbMediaSource, DataSource> sources = dataSources.remove(pipelineId);
    if (sources != null) {
        for (DataSource source : sources.values()) {
            try {
                // for filter to destroy custom datasource
                if (letHandlerDestroyIfSupport(pipelineId, source)) {
                    continue;
                }
                // fallback for regular destroy
                // TODO need to integrate to handler
                BasicDataSource basicDataSource = (BasicDataSource) source;
                basicDataSource.close();
            } catch (SQLException e) {
                logger.error("ERROR ## close the datasource has an error", e);
            }
        }
        sources.clear();
    }
}

Example 14 with DbMediaSource

use of com.alibaba.otter.shared.common.model.config.data.db.DbMediaSource in project otter by alibaba.

the class DbDialectIntegration method test_mysql.

@Test(expectedExceptions = RuntimeException.class)
public void test_mysql() {
    DbMediaSource dbMediaSource = new DbMediaSource();
    dbMediaSource.setId(10L);
    dbMediaSource.setDriver("com.mysql.jdbc.Driver");
    dbMediaSource.setUsername("xxxxx");
    dbMediaSource.setPassword("xxxxx");
    dbMediaSource.setUrl("jdbc:mysql://127.0.0.1:3306");
    dbMediaSource.setEncode("UTF-8");
    dbMediaSource.setType(DataMediaType.MYSQL);
    final DbDialect dbDialect = dbDialectFactory.getDbDialect(2L, dbMediaSource);
    want.object(dbDialect).clazIs(MysqlDialect.class);
    Table table = dbDialect.findTable("test", "ljh_demo");
    System.out.println(table);
    final SqlTemplate sqlTemplate = dbDialect.getSqlTemplate();
    final JdbcTemplate jdbcTemplate = dbDialect.getJdbcTemplate();
    final TransactionTemplate transactionTemplate = dbDialect.getTransactionTemplate();
    final int[] pkColumnTypes = { Types.INTEGER };
    final int[] columnTypes = { Types.VARCHAR, Types.INTEGER, Types.DECIMAL, Types.BIGINT };
    transactionTemplate.execute(new TransactionCallback() {

        public Object doInTransaction(TransactionStatus status) {
            int affect = 0;
            String sql = null;
            // 执行insert
            sql = sqlTemplate.getInsertSql(SCHEMA_NAME, TABLE_NAME, pkColumns, columns);
            System.out.println(sql);
            affect = (Integer) jdbcTemplate.execute(sql, new PreparedStatementCallback() {

                public Object doInPreparedStatement(PreparedStatement ps) throws SQLException, DataAccessException {
                    doPreparedStatement(ps, dbDialect, toTypes(columnTypes, pkColumnTypes), toValues(columnValues, pkColumnValues));
                    return ps.executeUpdate();
                }
            });
            want.number(affect).isEqualTo(1);
            throw new RuntimeException("rollback");
        }
    });
}

Example 15 with DbMediaSource

use of com.alibaba.otter.shared.common.model.config.data.db.DbMediaSource in project otter by alibaba.

the class DbPerfIntergration method test_stack.

@Test
public void test_stack() {
    DbMediaSource dbMediaSource = new DbMediaSource();
    dbMediaSource.setId(1L);
    dbMediaSource.setDriver("com.mysql.jdbc.Driver");
    dbMediaSource.setUsername("otter");
    dbMediaSource.setPassword("otter");
    dbMediaSource.setUrl("jdbc:mysql://127.0.0.1:3306/retl");
    dbMediaSource.setEncode("UTF-8");
    dbMediaSource.setType(DataMediaType.MYSQL);
    DbDataMedia dataMedia = new DbDataMedia();
    dataMedia.setSource(dbMediaSource);
    dataMedia.setId(1L);
    dataMedia.setName("ljhtable1");
    dataMedia.setNamespace("otter");
    final DbDialect dbDialect = dbDialectFactory.getDbDialect(2L, dataMedia.getSource());
    want.object(dbDialect).clazIs(MysqlDialect.class);
    final TransactionTemplate transactionTemplate = dbDialect.getTransactionTemplate();
    // 插入数据准备
    int minute = 5;
    int nextId = 1;
    final int thread = 10;
    final int batch = 50;
    final String sql = "insert into otter.ljhtable1 values(? , ? , ? , ?)";
    final CountDownLatch latch = new CountDownLatch(thread);
    ExecutorService executor = new ThreadPoolExecutor(thread, thread, 60, TimeUnit.SECONDS, new ArrayBlockingQueue(thread * 2), new NamedThreadFactory("load"), new ThreadPoolExecutor.CallerRunsPolicy());
    for (int sec = 0; sec < minute * 60; sec++) {
        // 执行秒循环
        long startTime = System.currentTimeMillis();
        for (int i = 0; i < thread; i++) {
            final int start = nextId + i * batch;
            executor.submit(new Runnable() {

                public void run() {
                    try {
                        transactionTemplate.execute(new TransactionCallback() {

                            public Object doInTransaction(TransactionStatus status) {
                                JdbcTemplate jdbcTemplate = dbDialect.getJdbcTemplate();
                                return jdbcTemplate.batchUpdate(sql, new BatchPreparedStatementSetter() {

                                    public void setValues(PreparedStatement ps, int idx) throws SQLException {
                                        int id = start + idx;
                                        StatementCreatorUtils.setParameterValue(ps, 1, Types.INTEGER, null, id);
                                        StatementCreatorUtils.setParameterValue(ps, 2, Types.VARCHAR, null, RandomStringUtils.randomAlphabetic(1000));
                                        // RandomStringUtils.randomAlphabetic()
                                        long time = new Date().getTime();
                                        StatementCreatorUtils.setParameterValue(ps, 3, Types.TIMESTAMP, new Timestamp(time));
                                        StatementCreatorUtils.setParameterValue(ps, 4, Types.TIMESTAMP, new Timestamp(time));
                                    }

                                    public int getBatchSize() {
                                        return batch;
                                    }
                                });
                            }
                        });
                    } finally {
                        latch.countDown();
                    }
                }
            });
        }
        long endTime = System.currentTimeMillis();
        try {
            latch.await(1000 * 60L - (endTime - startTime), TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        if (latch.getCount() != 0) {
            System.out.println("perf is not enough!");
            System.exit(-1);
        }
        endTime = System.currentTimeMillis();
        System.out.println("Time cost : " + (System.currentTimeMillis() - startTime));
        try {
            TimeUnit.MILLISECONDS.sleep(1000L - (endTime - startTime));
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        nextId = nextId + thread * batch;
    }
    executor.shutdown();
}