Examples with CsvSinkStreamOp com.alibaba.alink.operator.stream.sink.CsvSinkStreamOp used on opensource projects

Example 1 with CsvSinkStreamOp

use of com.alibaba.alink.operator.stream.sink.CsvSinkStreamOp in project Alink by alibaba.

the class CsvSourceSinkTest method testStreamCsvSinkAndSource.

@Category(DbTest.class)
@Test
public void testStreamCsvSinkAndSource() throws Exception {
    String filePath = path + "file2.csv";
    String[] columnNames = new String[] { "id", "content" };
    StreamOperator data = new MemSourceStreamOp(rows, columnNames);
    CsvSinkStreamOp sink = new CsvSinkStreamOp().setFilePath(filePath).setOverwriteSink(true);
    data.link(sink);
    StreamOperator.execute();
    List<String> lines = Files.readAllLines(Paths.get(filePath));
    lines.forEach(line -> {
        int pos = line.indexOf(',');
        String key = line.substring(0, pos);
        Assert.assertEquals(line, actual.get(key));
    });
    StreamOperator source = new CsvSourceStreamOp().setFilePath(filePath).setSchemaStr("id bigint, content string");
    source.print();
    StreamOperator.execute();
}

Example 2 with CsvSinkStreamOp

use of com.alibaba.alink.operator.stream.sink.CsvSinkStreamOp in project Alink by alibaba.

the class Chap03 method c_2_1_2.

static void c_2_1_2() throws Exception {
    HadoopFileSystem hdfs = new HadoopFileSystem(HADOOP_VERSION, HDFS_URI);
    OssFileSystem oss = new OssFileSystem(OSS_VERSION, OSS_END_POINT, OSS_BUCKET_NAME, OSS_ACCESS_ID, OSS_ACCESS_KEY);
    FilePath[] filePaths = new FilePath[] { new FilePath(LOCAL_DIR + "iris.csv"), new FilePath(HDFS_URI + "user/yangxu/alink/data/temp/iris.csv", hdfs), new FilePath(OSS_PREFIX_URI + "alink/data/temp/iris.csv", oss) };
    for (FilePath filePath : filePaths) {
        new CsvSourceBatchOp().setFilePath(IRIS_HTTP_URL).setSchemaStr(IRIS_SCHEMA_STR).link(new CsvSinkBatchOp().setFilePath(filePath).setOverwriteSink(true));
        BatchOperator.execute();
        System.out.println(new CsvSourceBatchOp().setFilePath(filePath).setSchemaStr(IRIS_SCHEMA_STR).count());
    }
    for (FilePath filePath : filePaths) {
        new CsvSourceStreamOp().setFilePath(IRIS_HTTP_URL).setSchemaStr(IRIS_SCHEMA_STR).link(new CsvSinkStreamOp().setFilePath(filePath).setOverwriteSink(true));
        StreamOperator.execute();
        new CsvSourceStreamOp().setFilePath(filePath).setSchemaStr(IRIS_SCHEMA_STR).filter("sepal_length < 4.5").print();
        StreamOperator.execute();
    }
}