Examples with LoadData io.mycat.sqlengine.mpp.LoadData used on opensource projects

Example 1 with LoadData

use of io.mycat.sqlengine.mpp.LoadData in project Mycat_plus by coderczp.

the class ServerLoadDataInfileHandler method parseOneLine.

private void parseOneLine(List<SQLExpr> columns, String tableName, String[] line, boolean toFile, String lineEnd) {
    RouteResultset rrs = tryDirectRoute(sql, line);
    if (rrs == null || rrs.getNodes() == null || rrs.getNodes().length == 0) {
        String insertSql = makeSimpleInsert(columns, line, tableName, true);
        rrs = serverConnection.routeSQL(insertSql, ServerParse.INSERT);
    }
    if (rrs == null || rrs.getNodes() == null || rrs.getNodes().length == 0) {
    // 无路由处理
    } else {
        for (RouteResultsetNode routeResultsetNode : rrs.getNodes()) {
            String name = routeResultsetNode.getName();
            LoadData data = routeResultMap.get(name);
            if (data == null) {
                data = new LoadData();
                data.setCharset(loadData.getCharset());
                data.setEnclose(loadData.getEnclose());
                data.setFieldTerminatedBy(loadData.getFieldTerminatedBy());
                data.setLineTerminatedBy(loadData.getLineTerminatedBy());
                data.setEscape(loadData.getEscape());
                routeResultMap.put(name, data);
            }
            String jLine = joinField(line, data);
            if (shoudAddSlot) {
                jLine = jLine + loadData.getFieldTerminatedBy() + routeResultsetNode.getSlot();
            }
            if (data.getData() == null) {
                data.setData(Lists.newArrayList(jLine));
            } else {
                data.getData().add(jLine);
            }
            if (toFile && // 避免当导入数据跨多分片时内存溢出的情况
            data.getData().size() > 10000) {
                saveDataToFile(data, name);
            }
        }
    }
}

Example 2 with LoadData

use of io.mycat.sqlengine.mpp.LoadData in project Mycat_plus by coderczp.

the class ServerLoadDataInfileHandler method parseLoadDataPram.

private void parseLoadDataPram() {
    loadData = new LoadData();
    SQLTextLiteralExpr rawLineEnd = (SQLTextLiteralExpr) statement.getLinesTerminatedBy();
    String lineTerminatedBy = rawLineEnd == null ? "\n" : rawLineEnd.getText();
    loadData.setLineTerminatedBy(lineTerminatedBy);
    SQLTextLiteralExpr rawFieldEnd = (SQLTextLiteralExpr) statement.getColumnsTerminatedBy();
    String fieldTerminatedBy = rawFieldEnd == null ? "\t" : rawFieldEnd.getText();
    loadData.setFieldTerminatedBy(fieldTerminatedBy);
    SQLTextLiteralExpr rawEnclosed = (SQLTextLiteralExpr) statement.getColumnsEnclosedBy();
    String enclose = rawEnclosed == null ? null : rawEnclosed.getText();
    loadData.setEnclose(enclose);
    SQLTextLiteralExpr escapseExpr = (SQLTextLiteralExpr) statement.getColumnsEscaped();
    String escapse = escapseExpr == null ? "\\" : escapseExpr.getText();
    loadData.setEscape(escapse);
    String charset = statement.getCharset() != null ? statement.getCharset() : serverConnection.getCharset();
    loadData.setCharset(charset);
    loadData.setFileName(fileName);
}

Example 3 with LoadData

use of io.mycat.sqlengine.mpp.LoadData in project Mycat_plus by coderczp.

the class ServerLoadDataInfileHandler method buildResultSet.

private RouteResultset buildResultSet(Map<String, LoadData> routeMap) {
    // 强制local
    statement.setLocal(true);
    // 默认druid会过滤掉路径的分隔符，所以这里重新设置下
    SQLLiteralExpr fn = new SQLCharExpr(fileName);
    statement.setFileName(fn);
    String srcStatement = statement.toString();
    RouteResultset rrs = new RouteResultset(srcStatement, ServerParse.LOAD_DATA_INFILE_SQL);
    rrs.setLoadData(true);
    rrs.setStatement(srcStatement);
    rrs.setAutocommit(serverConnection.isAutocommit());
    rrs.setFinishedRoute(true);
    int size = routeMap.size();
    RouteResultsetNode[] routeResultsetNodes = new RouteResultsetNode[size];
    int index = 0;
    for (String dn : routeMap.keySet()) {
        RouteResultsetNode rrNode = new RouteResultsetNode(dn, ServerParse.LOAD_DATA_INFILE_SQL, srcStatement);
        rrNode.setSource(rrs);
        rrNode.setTotalNodeSize(size);
        rrNode.setStatement(srcStatement);
        LoadData newLoadData = new LoadData();
        ObjectUtil.copyProperties(loadData, newLoadData);
        newLoadData.setLocal(true);
        LoadData loadData1 = routeMap.get(dn);
        // if (isHasStoreToFile)
        if (// 此处判断是否有保存分库load的临时文件dn1.txt/dn2.txt，不是判断是否有clientTemp.txt
        loadData1.getFileName() != null) {
            newLoadData.setFileName(loadData1.getFileName());
        } else {
            newLoadData.setData(loadData1.getData());
        }
        rrNode.setLoadData(newLoadData);
        routeResultsetNodes[index] = rrNode;
        index++;
    }
    rrs.setNodes(routeResultsetNodes);
    return rrs;
}

Example 4 with LoadData

use of io.mycat.sqlengine.mpp.LoadData in project Mycat-Server by MyCATApache.

the class ServerLoadDataInfileHandler method parseLoadDataPram.

private void parseLoadDataPram() {
    loadData = new LoadData();
    SQLTextLiteralExpr rawLineEnd = (SQLTextLiteralExpr) statement.getLinesTerminatedBy();
    String lineTerminatedBy = rawLineEnd == null ? "\n" : rawLineEnd.getText();
    loadData.setLineTerminatedBy(lineTerminatedBy);
    SQLTextLiteralExpr rawFieldEnd = (SQLTextLiteralExpr) statement.getColumnsTerminatedBy();
    String fieldTerminatedBy = rawFieldEnd == null ? "\t" : rawFieldEnd.getText();
    loadData.setFieldTerminatedBy(fieldTerminatedBy);
    SQLTextLiteralExpr rawEnclosed = (SQLTextLiteralExpr) statement.getColumnsEnclosedBy();
    String enclose = rawEnclosed == null ? null : rawEnclosed.getText();
    loadData.setEnclose(enclose);
    SQLTextLiteralExpr escapseExpr = (SQLTextLiteralExpr) statement.getColumnsEscaped();
    String escapse = escapseExpr == null ? "\\" : escapseExpr.getText();
    loadData.setEscape(escapse);
    String charset = statement.getCharset() != null ? statement.getCharset() : serverConnection.getCharset();
    loadData.setCharset(charset);
    loadData.setFileName(fileName);
}

Example 5 with LoadData

use of io.mycat.sqlengine.mpp.LoadData in project Mycat-Server by MyCATApache.

the class ServerLoadDataInfileHandler method buildResultSet.

private RouteResultset buildResultSet(Map<String, LoadData> routeMap) {
    // 强制local
    statement.setLocal(true);
    // 默认druid会过滤掉路径的分隔符，所以这里重新设置下
    SQLLiteralExpr fn = new SQLCharExpr(fileName);
    statement.setFileName(fn);
    String srcStatement = statement.toString();
    RouteResultset rrs = new RouteResultset(srcStatement, ServerParse.LOAD_DATA_INFILE_SQL);
    rrs.setLoadData(true);
    rrs.setStatement(srcStatement);
    rrs.setAutocommit(serverConnection.isAutocommit());
    rrs.setFinishedRoute(true);
    int size = routeMap.size();
    RouteResultsetNode[] routeResultsetNodes = new RouteResultsetNode[size];
    int index = 0;
    for (String dn : routeMap.keySet()) {
        RouteResultsetNode rrNode = new RouteResultsetNode(dn, ServerParse.LOAD_DATA_INFILE_SQL, srcStatement);
        rrNode.setSource(rrs);
        rrNode.setTotalNodeSize(size);
        rrNode.setStatement(srcStatement);
        LoadData newLoadData = new LoadData();
        ObjectUtil.copyProperties(loadData, newLoadData);
        newLoadData.setLocal(true);
        LoadData loadData1 = routeMap.get(dn);
        // if (isHasStoreToFile)
        if (// 此处判断是否有保存分库load的临时文件dn1.txt/dn2.txt，不是判断是否有clientTemp.txt
        loadData1.getFileName() != null) {
            newLoadData.setFileName(loadData1.getFileName());
        } else {
            newLoadData.setData(loadData1.getData());
        }
        rrNode.setLoadData(newLoadData);
        routeResultsetNodes[index] = rrNode;
        index++;
    }
    rrs.setNodes(routeResultsetNodes);
    return rrs;
}