Examples with LoadData com.actiontech.dble.sqlengine.mpp.LoadData used on opensource projects

Example 1 with LoadData

use of com.actiontech.dble.sqlengine.mpp.LoadData in project dble by actiontech.

the class ServerLoadDataInfileHandler method parseLoadDataPram.

private void parseLoadDataPram() {
    loadData = new LoadData();
    SQLTextLiteralExpr rawLineEnd = (SQLTextLiteralExpr) statement.getLinesTerminatedBy();
    String lineTerminatedBy = rawLineEnd == null ? "\n" : rawLineEnd.getText();
    loadData.setLineTerminatedBy(lineTerminatedBy);
    SQLTextLiteralExpr rawFieldEnd = (SQLTextLiteralExpr) statement.getColumnsTerminatedBy();
    String fieldTerminatedBy = rawFieldEnd == null ? "\t" : rawFieldEnd.getText();
    loadData.setFieldTerminatedBy(fieldTerminatedBy);
    SQLTextLiteralExpr rawEnclosed = (SQLTextLiteralExpr) statement.getColumnsEnclosedBy();
    String enclose = rawEnclosed == null ? null : rawEnclosed.getText();
    loadData.setEnclose(enclose);
    SQLTextLiteralExpr escapedExpr = (SQLTextLiteralExpr) statement.getColumnsEscaped();
    String escaped = escapedExpr == null ? "\\" : escapedExpr.getText();
    loadData.setEscape(escaped);
    String charset = statement.getCharset() != null ? statement.getCharset() : DbleServer.getInstance().getSystemVariables().getDefaultValue("character_set_database");
    loadData.setCharset(charset);
    loadData.setFileName(fileName);
}

Example 2 with LoadData

use of com.actiontech.dble.sqlengine.mpp.LoadData in project dble by actiontech.

the class ServerLoadDataInfileHandler method buildResultSet.

private RouteResultset buildResultSet(Map<String, LoadData> routeMap) {
    statement.setLocal(true);
    // druid will filter path, reset it now
    SQLLiteralExpr fn = new SQLCharExpr(fileName);
    statement.setFileName(fn);
    // replace IGNORE X LINES in SQL to avoid  IGNORING X LINE in every node.
    String srcStatement = this.ignoreLinesDelete(statement.toString());
    RouteResultset rrs = new RouteResultset(srcStatement, ServerParse.LOAD_DATA_INFILE_SQL);
    rrs.setLoadData(true);
    rrs.setStatement(srcStatement);
    rrs.setFinishedRoute(true);
    int size = routeMap.size();
    RouteResultsetNode[] routeResultsetNodes = new RouteResultsetNode[size];
    int index = 0;
    for (Map.Entry<String, LoadData> entry : routeMap.entrySet()) {
        RouteResultsetNode rrNode = new RouteResultsetNode(entry.getKey(), ServerParse.LOAD_DATA_INFILE_SQL, srcStatement);
        rrNode.setStatement(srcStatement);
        LoadData newLoadData = new LoadData();
        ObjectUtil.copyProperties(loadData, newLoadData);
        newLoadData.setLocal(true);
        LoadData loadData1 = entry.getValue();
        if (loadData1.getFileName() != null) {
            newLoadData.setFileName(loadData1.getFileName());
        } else {
            newLoadData.setData(loadData1.getData());
        }
        rrNode.setLoadData(newLoadData);
        routeResultsetNodes[index] = rrNode;
        index++;
    }
    rrs.setNodes(routeResultsetNodes);
    return rrs;
}

Example 3 with LoadData

use of com.actiontech.dble.sqlengine.mpp.LoadData in project dble by actiontech.

the class ServerLoadDataInfileHandler method parseOneLine.

private void parseOneLine(List<SQLExpr> columns, String table, String[] line, boolean toFile, String lineEnd) {
    if (loadData.getEnclose() != null && loadData.getEnclose().charAt(0) > 0x0020) {
        for (int i = 0; i < line.length; i++) {
            line[i] = line[i].trim();
        }
    }
    RouteResultset rrs = tryDirectRoute(sql, line);
    if (rrs == null || rrs.getNodes() == null || rrs.getNodes().length == 0) {
        String insertSql = makeSimpleInsert(columns, line, table);
        rrs = serverConnection.routeSQL(insertSql, ServerParse.INSERT);
    }
    if (rrs == null || rrs.getNodes() == null || rrs.getNodes().length == 0) {
    // do nothing
    } else {
        for (RouteResultsetNode routeResultsetNode : rrs.getNodes()) {
            String name = routeResultsetNode.getName();
            LoadData data = routeResultMap.get(name);
            if (data == null) {
                data = new LoadData();
                data.setCharset(loadData.getCharset());
                data.setEnclose(loadData.getEnclose());
                data.setFieldTerminatedBy(loadData.getFieldTerminatedBy());
                data.setLineTerminatedBy(loadData.getLineTerminatedBy());
                data.setEscape(loadData.getEscape());
                routeResultMap.put(name, data);
            }
            String jLine = joinField(line, data);
            if (data.getData() == null) {
                data.setData(Lists.newArrayList(jLine));
            } else {
                data.getData().add(jLine);
            }
            if (toFile && data.getData().size() > 10000) {
                // avoid OOM
                saveDataToFile(data, name);
            }
        }
    }
}

Example 4 with LoadData

use of com.actiontech.dble.sqlengine.mpp.LoadData in project dble by actiontech.

the class LoadDataUtil method requestFileDataResponse.

public static void requestFileDataResponse(byte[] data, BackendConnection conn) {
    byte packId = data[3];
    BackendAIOConnection backendAIOConnection = (BackendAIOConnection) conn;
    RouteResultsetNode rrn = (RouteResultsetNode) conn.getAttachment();
    LoadData loadData = rrn.getLoadData();
    List<String> loadDataData = loadData.getData();
    try {
        if (loadDataData != null && loadDataData.size() > 0) {
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            for (int i = 0, loadDataDataSize = loadDataData.size(); i < loadDataDataSize; i++) {
                String line = loadDataData.get(i);
                String s = (i == loadDataDataSize - 1) ? line : line + loadData.getLineTerminatedBy();
                byte[] bytes = s.getBytes(CharsetUtil.getJavaCharset(loadData.getCharset()));
                bos.write(bytes);
            }
            packId = writeToBackConnection(packId, new ByteArrayInputStream(bos.toByteArray()), backendAIOConnection);
        } else {
            packId = writeToBackConnection(packId, new BufferedInputStream(new FileInputStream(loadData.getFileName())), backendAIOConnection);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        // send empty packet
        byte[] empty = new byte[] { 0, 0, 0, 3 };
        empty[3] = ++packId;
        backendAIOConnection.write(empty);
    }
}