Examples with LoadData io.mycat.sqlengine.mpp.LoadData used on opensource projects

Example 6 with LoadData

use of io.mycat.sqlengine.mpp.LoadData in project Mycat_plus by coderczp.

the class LoadDataUtil method requestFileDataResponse.

public static void requestFileDataResponse(byte[] data, BackendConnection conn) {
    byte packId = data[3];
    BackendAIOConnection backendAIOConnection = (BackendAIOConnection) conn;
    RouteResultsetNode rrn = (RouteResultsetNode) conn.getAttachment();
    LoadData loadData = rrn.getLoadData();
    List<String> loadDataData = loadData.getData();
    try {
        if (loadDataData != null && loadDataData.size() > 0) {
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            for (int i = 0, loadDataDataSize = loadDataData.size(); i < loadDataDataSize; i++) {
                String line = loadDataData.get(i);
                String s = (i == loadDataDataSize - 1) ? line : line + loadData.getLineTerminatedBy();
                byte[] bytes = s.getBytes(loadData.getCharset());
                bos.write(bytes);
            }
            packId = writeToBackConnection(packId, new ByteArrayInputStream(bos.toByteArray()), backendAIOConnection);
        } else {
            // 从文件读取
            packId = writeToBackConnection(packId, new BufferedInputStream(new FileInputStream(loadData.getFileName())), backendAIOConnection);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        // 结束必须发空包
        byte[] empty = new byte[] { 0, 0, 0, 3 };
        empty[3] = ++packId;
        backendAIOConnection.write(empty);
    }
}

Example 7 with LoadData

use of io.mycat.sqlengine.mpp.LoadData in project Mycat-Server by MyCATApache.

the class ServerLoadDataInfileHandler method parseOneLine.

private void parseOneLine(List<SQLExpr> columns, String tableName, String[] line, boolean toFile, String lineEnd) {
    RouteResultset rrs = tryDirectRoute(sql, line);
    if (rrs == null || rrs.getNodes() == null || rrs.getNodes().length == 0) {
        String insertSql = makeSimpleInsert(columns, line, tableName, true);
        rrs = serverConnection.routeSQL(insertSql, ServerParse.INSERT);
    }
    if (rrs == null || rrs.getNodes() == null || rrs.getNodes().length == 0) {
    // 无路由处理
    } else {
        for (RouteResultsetNode routeResultsetNode : rrs.getNodes()) {
            String name = routeResultsetNode.getName();
            LoadData data = routeResultMap.get(name);
            if (data == null) {
                data = new LoadData();
                data.setCharset(loadData.getCharset());
                data.setEnclose(loadData.getEnclose());
                data.setFieldTerminatedBy(loadData.getFieldTerminatedBy());
                data.setLineTerminatedBy(loadData.getLineTerminatedBy());
                data.setEscape(loadData.getEscape());
                routeResultMap.put(name, data);
            }
            String jLine = joinField(line, data);
            if (shoudAddSlot) {
                jLine = jLine + loadData.getFieldTerminatedBy() + routeResultsetNode.getSlot();
            }
            if (data.getData() == null) {
                data.setData(Lists.newArrayList(jLine));
            } else {
                data.getData().add(jLine);
            }
            if (toFile && // 避免当导入数据跨多分片时内存溢出的情况
            data.getData().size() > 10000) {
                saveDataToFile(data, name);
            }
        }
    }
}

Example 8 with LoadData

use of io.mycat.sqlengine.mpp.LoadData in project Mycat-Server by MyCATApache.

the class LoadDataUtil method requestFileDataResponse.

public static void requestFileDataResponse(byte[] data, BackendConnection conn) {
    byte packId = data[3];
    BackendAIOConnection backendAIOConnection = (BackendAIOConnection) conn;
    RouteResultsetNode rrn = (RouteResultsetNode) conn.getAttachment();
    LoadData loadData = rrn.getLoadData();
    List<String> loadDataData = loadData.getData();
    try {
        if (loadDataData != null && loadDataData.size() > 0) {
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            for (int i = 0, loadDataDataSize = loadDataData.size(); i < loadDataDataSize; i++) {
                String line = loadDataData.get(i);
                String s = (i == loadDataDataSize - 1) ? line : line + loadData.getLineTerminatedBy();
                byte[] bytes = s.getBytes(loadData.getCharset());
                bos.write(bytes);
            }
            packId = writeToBackConnection(packId, new ByteArrayInputStream(bos.toByteArray()), backendAIOConnection);
        } else {
            // 从文件读取
            packId = writeToBackConnection(packId, new BufferedInputStream(new FileInputStream(loadData.getFileName())), backendAIOConnection);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        // 结束必须发空包
        byte[] empty = new byte[] { 0, 0, 0, 3 };
        empty[3] = ++packId;
        backendAIOConnection.write(empty);
    }
}