Examples with RelationalResultToCSVSerializer org.finos.legend.engine.plan.execution.stores.relational.serialization.RelationalResultToCSVSerializer used on opensource projects

Example 1 with RelationalResultToCSVSerializer

use of org.finos.legend.engine.plan.execution.stores.relational.serialization.RelationalResultToCSVSerializer in project legend-engine by finos.

the class TestRelationalExecutor method testGMTTimeZonePlanWithDifferentSerializationFormats.

@Test
public void testGMTTimeZonePlanWithDifferentSerializationFormats() throws IOException {
    String planString = "{\"_type\":\"simple\",\"authDependent\":false,\"kerberos\":null,\"serializer\":{\"name\":\"pure\",\"version\":\"vX_X_X\"},\"templateFunctions\":[\"<#function renderCollection collection separator><#return collection?join(separator)></#function>\",\"<#function collectionSize collection> <#return collection?size> </#function>\"],\"rootExecutionNode\":{\"_type\":\"relationalTdsInstantiation\",\"resultType\":{\"_type\":\"tds\",\"tdsColumns\":[{\"name\":\"testDateTime\",\"type\":\"DateTime\",\"doc\":null,\"relationalType\":\"TIMESTAMP\",\"enumMapping\":{}}]},\"executionNodes\":[{\"_type\":\"sql\",\"resultType\":{\"_type\":\"dataType\",\"dataType\":\"meta::pure::metamodel::type::Any\"},\"executionNodes\":[],\"resultSizeRange\":null,\"implementation\":null,\"sqlQuery\":\"select \\\"root\\\".testDateTime as \\\"testDateTime\\\" from ProductSynonymTable as \\\"root\\\"\",\"onConnectionCloseCommitQuery\":null,\"onConnectionCloseRollbackQuery\":null,\"connection\":{\"_type\": \"RelationalDatabaseConnection\",\"type\": \"H2\",\"authenticationStrategy\" : {\"_type\" : \"test\"},\"datasourceSpecification\" : {\"_type\" : \"static\",\"databaseName\" : \"testDB\",\"host\":\"127.0.0.1\",\"port\" : \"" + serverPort + "\"},\"timeZone\":\"GMT\"},\"resultColumns\":[{\"label\":\"\\\"testDateTime\\\"\",\"dataType\":\"TIMESTAMP\"}]}],\"resultSizeRange\":null,\"implementation\":null},\"globalImplementationSupport\":null}";
    SingleExecutionPlan plan = objectMapper.readValue(planString, SingleExecutionPlan.class);
    RelationalResult result_default = (RelationalResult) plan.rootExecutionNode.accept(new ExecutionNodeExecutor(null, new ExecutionState(Maps.mutable.empty(), Lists.mutable.withAll(plan.templateFunctions), Lists.mutable.with(new RelationalStoreExecutionState(new RelationalStoreState(serverPort))))));
    Assert.assertEquals("{\"builder\": {\"_type\":\"tdsBuilder\",\"columns\":[{\"name\":\"testDateTime\",\"type\":\"DateTime\",\"relationalType\":\"TIMESTAMP\"}]}, \"activities\": [{\"_type\":\"relational\",\"sql\":\"select \\\"root\\\".testDateTime as \\\"testDateTime\\\" from ProductSynonymTable as \\\"root\\\"\"}], \"result\" : {\"columns\" : [\"testDateTime\"], \"rows\" : [{\"values\": [\"2014-12-04T15:22:23.123456789+0000\"]},{\"values\": [\"2014-12-04T23:22:23.123456789+0000\"]},{\"values\": [\"2014-12-04T08:22:23.000000000+0000\"]},{\"values\": [\"2014-12-04T08:22:23.123000000+0000\"]},{\"values\": [\"2013-12-04T10:22:23.000000000+0000\"]},{\"values\": [\"2013-04-04T08:22:23.123000000+0000\"]}]}}", result_default.flush(new RelationalResultToJsonDefaultSerializer(result_default)));
    RelationalResult result_pureTdsObject = (RelationalResult) plan.rootExecutionNode.accept(new ExecutionNodeExecutor(null, new ExecutionState(Maps.mutable.empty(), Lists.mutable.withAll(plan.templateFunctions), Lists.mutable.with(new RelationalStoreExecutionState(new RelationalStoreState(serverPort))))));
    Assert.assertEquals("[{\"testDateTime\":\"2014-12-04T15:22:23.123456789+0000\"},{\"testDateTime\":\"2014-12-04T23:22:23.123456789+0000\"},{\"testDateTime\":\"2014-12-04T08:22:23.000000000+0000\"},{\"testDateTime\":\"2014-12-04T08:22:23.123000000+0000\"},{\"testDateTime\":\"2013-12-04T10:22:23.000000000+0000\"},{\"testDateTime\":\"2013-04-04T08:22:23.123000000+0000\"}]", result_pureTdsObject.flush(new RelationalResultToPureTDSToObjectSerializer(result_pureTdsObject)));
    RelationalResult result_pureTds = (RelationalResult) plan.rootExecutionNode.accept(new ExecutionNodeExecutor(null, new ExecutionState(Maps.mutable.empty(), Lists.mutable.withAll(plan.templateFunctions), Lists.mutable.with(new RelationalStoreExecutionState(new RelationalStoreState(serverPort))))));
    Assert.assertEquals("{\"columns\":[{\"name\":\"testDateTime\",\"type\":\"DateTime\"}],\"rows\":[{\"values\":[\"2014-12-04T15:22:23.123456789+0000\"]},{\"values\":[\"2014-12-04T23:22:23.123456789+0000\"]},{\"values\":[\"2014-12-04T08:22:23.000000000+0000\"]},{\"values\":[\"2014-12-04T08:22:23.123000000+0000\"]},{\"values\":[\"2013-12-04T10:22:23.000000000+0000\"]},{\"values\":[\"2013-04-04T08:22:23.123000000+0000\"]}]}", result_pureTds.flush(new RelationalResultToPureTDSSerializer(result_pureTds)));
    RelationalResult result_jsonTds = (RelationalResult) plan.rootExecutionNode.accept(new ExecutionNodeExecutor(null, new ExecutionState(Maps.mutable.empty(), Lists.mutable.withAll(plan.templateFunctions), Lists.mutable.with(new RelationalStoreExecutionState(new RelationalStoreState(serverPort))))));
    Assert.assertEquals("{\"columns\":[{\"name\":\"testDateTime\",\"type\":\"DateTime\",\"relationalType\":\"TIMESTAMP\"}],\"rows\":[[\"2014-12-04T15:22:23.123456789+0000\"],[\"2014-12-04T23:22:23.123456789+0000\"],[\"2014-12-04T08:22:23.000000000+0000\"],[\"2014-12-04T08:22:23.123000000+0000\"],[\"2013-12-04T10:22:23.000000000+0000\"],[\"2013-04-04T08:22:23.123000000+0000\"]]}", result_jsonTds.flush(new JSONTDSSerializer(result_jsonTds, false, false)));
    RelationalResult result_csv = (RelationalResult) plan.rootExecutionNode.accept(new ExecutionNodeExecutor(null, new ExecutionState(Maps.mutable.empty(), Lists.mutable.withAll(plan.templateFunctions), Lists.mutable.with(new RelationalStoreExecutionState(new RelationalStoreState(serverPort))))));
    Assert.assertEquals("2014-12-04 15:22:23.123456789\r\n" + "2014-12-04 23:22:23.123456789\r\n" + "2014-12-04 08:22:23.0\r\n" + "2014-12-04 08:22:23.123\r\n" + "2013-12-04 10:22:23.0\r\n" + "2013-04-04 08:22:23.123\r\n", result_csv.flush(new RelationalResultToCSVSerializer(result_csv)));
}

Example 2 with RelationalResultToCSVSerializer

use of org.finos.legend.engine.plan.execution.stores.relational.serialization.RelationalResultToCSVSerializer in project legend-engine by finos.

the class StreamResultToTempTableVisitor method visit.

@Override
public Boolean visit(H2Commands h2Commands) {
    if (ingestionMethod == null) {
        ingestionMethod = h2Commands.getDefaultIngestionMethod();
    }
    if (ingestionMethod == IngestionMethod.CLIENT_FILE) {
        try (TemporaryFile tempFile = new TemporaryFile(config.tempPath)) {
            CsvSerializer csvSerializer;
            if (result instanceof RelationalResult) {
                csvSerializer = new RelationalResultToCSVSerializer((RelationalResult) result, true);
                tempFile.writeFile(csvSerializer);
                try (Statement statement = connection.createStatement()) {
                    statement.execute(h2Commands.dropTempTable(tableName));
                    RelationalResult relationalResult = (RelationalResult) result;
                    if (result.getResultBuilder() instanceof TDSBuilder) {
                        h2Commands.createAndLoadTempTable(tableName, relationalResult.getTdsColumns().stream().map(c -> new Column(c.name, c.relationalType)).collect(Collectors.toList()), tempFile.getTemporaryPathForFile()).forEach(x -> checkedExecute(statement, x));
                    } else {
                        h2Commands.createAndLoadTempTable(tableName, relationalResult.getSQLResultColumns().stream().map(c -> new Column(c.label, c.dataType)).collect(Collectors.toList()), tempFile.getTemporaryPathForFile()).forEach(x -> checkedExecute(statement, x));
                    }
                }
            } else if (result instanceof RealizedRelationalResult) {
                csvSerializer = new RealizedRelationalResultCSVSerializer((RealizedRelationalResult) result, this.databaseTimeZone, true, false);
                tempFile.writeFile(csvSerializer);
                try (Statement statement = connection.createStatement()) {
                    statement.execute(h2Commands.dropTempTable(tableName));
                    RealizedRelationalResult realizedRelationalResult = (RealizedRelationalResult) result;
                    h2Commands.createAndLoadTempTable(tableName, realizedRelationalResult.columns.stream().map(c -> new Column(c.label, c.dataType)).collect(Collectors.toList()), tempFile.getTemporaryPathForFile()).forEach(x -> checkedExecute(statement, x));
                }
            } else if (result instanceof StreamingObjectResult) {
                csvSerializer = new StreamingObjectResultCSVSerializer((StreamingObjectResult) result, true);
                tempFile.writeFile(csvSerializer);
                try (Statement statement = connection.createStatement()) {
                    statement.execute(h2Commands.dropTempTable(tableName));
                    h2Commands.createAndLoadTempTable(tableName, csvSerializer.getHeaderColumnsAndTypes().stream().map(c -> new Column(c.getOne(), RelationalExecutor.getRelationalTypeFromDataType(c.getTwo()))).collect(Collectors.toList()), tempFile.getTemporaryPathForFile()).forEach(x -> checkedExecute(statement, x));
                }
            } else if (result instanceof TempTableStreamingResult) {
                csvSerializer = new StreamingTempTableResultCSVSerializer((TempTableStreamingResult) result, true);
                tempFile.writeFile(csvSerializer);
                try (Statement statement = connection.createStatement()) {
                    statement.execute(h2Commands.dropTempTable(tableName));
                    h2Commands.createAndLoadTempTable(tableName, csvSerializer.getHeaderColumnsAndTypes().stream().map(c -> new Column(c.getOne(), RelationalExecutor.getRelationalTypeFromDataType(c.getTwo()))).collect(Collectors.toList()), tempFile.getTemporaryPathForFile()).forEach(x -> checkedExecute(statement, x));
                }
            } else {
                throw new RuntimeException("Result not supported yet: " + result.getClass().getName());
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    } else if (ingestionMethod == IngestionMethod.BATCH_INSERT) {
        streamResultToNewTarget(((RelationalResult) result).resultSet, connection, tableName, 100);
    }
    return true;
}