Examples with PhysicalDatasource io.mycat.backend.datasource.PhysicalDatasource used on opensource projects

Example 46 with PhysicalDatasource

use of io.mycat.backend.datasource.PhysicalDatasource in project Mycat-Server by MyCATApache.

the class MigrateDumpRunner method run.

@Override
public void run() {
    try {
        String mysqldump = "?mysqldump -h? -P? -u? -p?  ? ? --single-transaction -q --default-character-set=utf8mb4 --hex-blob --where=\"?\" --master-data=1  -T  \"?\"  --fields-enclosed-by=\\\" --fields-terminated-by=, --lines-terminated-by=\\n  --fields-escaped-by=\\\\ ";
        PhysicalDBPool dbPool = MycatServer.getInstance().getConfig().getDataNodes().get(task.getFrom()).getDbPool();
        PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()];
        DBHostConfig config = datasource.getConfig();
        File file = null;
        String spath = querySecurePath(config);
        if (Strings.isNullOrEmpty(spath) || "NULL".equalsIgnoreCase(spath) || "empty".equalsIgnoreCase(spath)) {
            file = new File(SystemConfig.getHomePath() + File.separator + "temp", "dump");
        // task.getFrom() + "_" + task.getTo() + Thread.currentThread().getId() + System.currentTimeMillis() + "");
        } else {
            spath += Thread.currentThread().getId() + System.currentTimeMillis();
            file = new File(spath);
        }
        file.mkdirs();
        String encose = OperatingSystem.isWindows() ? "\\" : "";
        String finalCmd = DataMigratorUtil.paramsAssignment(mysqldump, "?", "", config.getIp(), String.valueOf(config.getPort()), config.getUser(), config.getPassword(), MigrateUtils.getDatabaseFromDataNode(task.getFrom()), task.getTable(), makeWhere(task), file.getPath());
        List<String> args = Arrays.asList("mysqldump", "-h" + config.getIp(), "-P" + String.valueOf(config.getPort()), "-u" + config.getUser(), "-p" + config.getPassword(), MigrateUtils.getDatabaseFromDataNode(task.getFrom()), task.getTable(), "--single-transaction", "-q", "--default-character-set=utf8mb4", "--hex-blob", "--where=" + makeWhere(task), "--master-data=1", "-T" + file.getPath(), "--fields-enclosed-by=" + encose + "\"", "--fields-terminated-by=,", "--lines-terminated-by=\\n", "--fields-escaped-by=\\\\");
        String result = ProcessUtil.execReturnString(args);
        int logIndex = result.indexOf("MASTER_LOG_FILE='");
        int logPosIndex = result.indexOf("MASTER_LOG_POS=");
        String logFile = result.substring(logIndex + 17, logIndex + 17 + result.substring(logIndex + 17).indexOf("'"));
        String logPos = result.substring(logPosIndex + 15, logPosIndex + 15 + result.substring(logPosIndex + 15).indexOf(";"));
        task.setBinlogFile(logFile);
        task.setPos(Integer.parseInt(logPos));
        File dataFile = new File(file, task.getTable() + ".txt");
        File sqlFile = new File(file, task.getTable() + ".sql");
        List<String> createTable = Files.readLines(sqlFile, Charset.forName("UTF-8"));
        exeCreateTableToDn(extractCreateSql(createTable), task.getTo(), task.getTable());
        if (dataFile.length() > 0) {
            loaddataToDn(dataFile, task.getTo(), task.getTable());
        }
        pushMsgToZK(task.getZkpath(), task.getFrom() + "-" + task.getTo(), 1, "sucess", logFile, logPos);
        DataMigratorUtil.deleteDir(file);
        sucessTask.getAndIncrement();
    } catch (Exception e) {
        try {
            pushMsgToZK(task.getZkpath(), task.getFrom() + "-" + task.getTo(), 0, e.getMessage(), "", "");
        } catch (Exception e1) {
        }
        LOGGER.error("error:", e);
    } finally {
        latch.countDown();
    }
}

Example 47 with PhysicalDatasource

use of io.mycat.backend.datasource.PhysicalDatasource in project Mycat-Server by MyCATApache.

the class MigrateDumpRunner method loaddataToDn.

private void loaddataToDn(File loaddataFile, String toDn, String table) throws SQLException, IOException {
    PhysicalDBNode dbNode = MycatServer.getInstance().getConfig().getDataNodes().get(toDn);
    PhysicalDBPool dbPool = dbNode.getDbPool();
    PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()];
    DBHostConfig config = datasource.getConfig();
    Connection con = null;
    try {
        con = DriverManager.getConnection("jdbc:mysql://" + config.getUrl() + "/" + dbNode.getDatabase(), config.getUser(), config.getPassword());
        String sql = "load data local infile '" + loaddataFile.getPath().replace("\\", "//") + "' replace into table " + table + " character set 'utf8mb4'  fields terminated by ','  enclosed by '\"'  ESCAPED BY '\\\\'  lines terminated by '\\n'";
        JdbcUtils.execute(con, sql, new ArrayList<>());
    } finally {
        JdbcUtils.close(con);
    }
}

Example 48 with PhysicalDatasource

use of io.mycat.backend.datasource.PhysicalDatasource in project Mycat-Server by MyCATApache.

the class MigrateDumpRunner method exeCreateTableToDn.

private void exeCreateTableToDn(String sql, String toDn, String table) throws SQLException {
    PhysicalDBNode dbNode = MycatServer.getInstance().getConfig().getDataNodes().get(toDn);
    PhysicalDBPool dbPool = dbNode.getDbPool();
    PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()];
    DBHostConfig config = datasource.getConfig();
    Connection con = null;
    try {
        con = DriverManager.getConnection("jdbc:mysql://" + config.getUrl() + "/" + dbNode.getDatabase(), config.getUser(), config.getPassword());
        JdbcUtils.execute(con, sql, new ArrayList<>());
    } finally {
        JdbcUtils.close(con);
    }
}

Example 49 with PhysicalDatasource

use of io.mycat.backend.datasource.PhysicalDatasource in project Mycat-Server by MyCATApache.

the class MigrateMainRunner method run.

@Override
public void run() {
    AtomicInteger sucessTask = new AtomicInteger(0);
    CountDownLatch downLatch = new CountDownLatch(migrateTaskList.size());
    for (MigrateTask migrateTask : migrateTaskList) {
        MycatServer.getInstance().getBusinessExecutor().submit(new MigrateDumpRunner(migrateTask, downLatch, sucessTask));
    }
    try {
        downLatch.await(2, TimeUnit.HOURS);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
    // 同一个dataHost的任务合并执行，避免过多流量浪费
    if (sucessTask.get() == migrateTaskList.size()) {
        long binlogFileNum = -1;
        String binlogFile = "";
        long pos = -1;
        for (MigrateTask migrateTask : migrateTaskList) {
            if (binlogFileNum == -1) {
                binlogFileNum = Integer.parseInt(migrateTask.getBinlogFile().substring(migrateTask.getBinlogFile().lastIndexOf(".") + 1));
                binlogFile = migrateTask.getBinlogFile();
                pos = migrateTask.getPos();
            } else {
                int tempBinlogFileNum = Integer.parseInt(migrateTask.getBinlogFile().substring(migrateTask.getBinlogFile().lastIndexOf(".") + 1));
                if (tempBinlogFileNum <= binlogFileNum && migrateTask.getPos() <= pos) {
                    binlogFileNum = tempBinlogFileNum;
                    binlogFile = migrateTask.getBinlogFile();
                    pos = migrateTask.getPos();
                }
            }
        }
        String taskPath = migrateTaskList.get(0).getZkpath();
        taskPath = taskPath.substring(0, taskPath.lastIndexOf("/"));
        String taskID = taskPath.substring(taskPath.lastIndexOf('/') + 1, taskPath.length());
        // 开始增量数据迁移
        PhysicalDBPool dbPool = MycatServer.getInstance().getConfig().getDataHosts().get(dataHost);
        PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()];
        DBHostConfig config = datasource.getConfig();
        BinlogStream stream = new BinlogStream(config.getUrl().substring(0, config.getUrl().indexOf(":")), config.getPort(), config.getUser(), config.getPassword());
        try {
            stream.setSlaveID(migrateTaskList.get(0).getSlaveId());
            stream.setBinglogFile(binlogFile);
            stream.setBinlogPos(pos);
            stream.setMigrateTaskList(migrateTaskList);
            BinlogStreamHoder.binlogStreamMap.put(taskID, stream);
            stream.connect();
        } catch (IOException e) {
            LOGGER.error("error:", e);
        }
    }
}

Example 50 with PhysicalDatasource

use of io.mycat.backend.datasource.PhysicalDatasource in project Mycat-Server by MyCATApache.

the class PostgreSQLHeartbeat method switchSourceIfNeed.

private void switchSourceIfNeed(String reason) {
    int switchType = source.getHostConfig().getSwitchType();
    if (switchType == DataHostConfig.NOT_SWITCH_DS) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("not switch datasource ,for switchType is " + DataHostConfig.NOT_SWITCH_DS);
            return;
        }
        return;
    }
    PhysicalDBPool pool = this.source.getDbPool();
    int curDatasourceHB = pool.getSource().getHeartbeat().getStatus();
    // read node can't switch ,only write node can switch
    if (pool.getWriteType() == PhysicalDBPool.WRITE_ONLYONE_NODE && !source.isReadNode() && curDatasourceHB != DBHeartbeat.OK_STATUS && pool.getSources().length > 1) {
        synchronized (pool) {
            // try to see if need switch datasource
            curDatasourceHB = pool.getSource().getHeartbeat().getStatus();
            if (curDatasourceHB != DBHeartbeat.INIT_STATUS && curDatasourceHB != DBHeartbeat.OK_STATUS) {
                int curIndex = pool.getActivedIndex();
                int nextId = pool.next(curIndex);
                PhysicalDatasource[] allWriteNodes = pool.getSources();
                while (true) {
                    if (nextId == curIndex) {
                        break;
                    }
                    PhysicalDatasource theSource = allWriteNodes[nextId];
                    DBHeartbeat theSourceHB = theSource.getHeartbeat();
                    int theSourceHBStatus = theSourceHB.getStatus();
                    if (theSourceHBStatus == DBHeartbeat.OK_STATUS) {
                        if (switchType == DataHostConfig.SYN_STATUS_SWITCH_DS) {
                            if (Integer.valueOf(0).equals(theSourceHB.getSlaveBehindMaster())) {
                                LOGGER.info("try to switch datasource ,slave is synchronized to master " + theSource.getConfig());
                                pool.switchSource(nextId, true, reason);
                                break;
                            } else {
                                LOGGER.warn("ignored  datasource ,slave is not  synchronized to master , slave behind master :" + theSourceHB.getSlaveBehindMaster() + " " + theSource.getConfig());
                            }
                        } else {
                            // normal switch
                            LOGGER.info("try to switch datasource ,not checked slave synchronize status " + theSource.getConfig());
                            pool.switchSource(nextId, true, reason);
                            break;
                        }
                    }
                    nextId = pool.next(nextId);
                }
            }
        }
    }
}