Examples with CollectSinkStreamOp com.alibaba.alink.operator.stream.sink.CollectSinkStreamOp used on opensource projects

Example 1 with CollectSinkStreamOp

use of com.alibaba.alink.operator.stream.sink.CollectSinkStreamOp in project Alink by alibaba.

the class SoftmaxTest method pipelineTest.

@Test
public void pipelineTest() throws Exception {
    BatchOperator<?> vecdata = new MemSourceBatchOp(Arrays.asList(vecrows), veccolNames);
    StreamOperator<?> svecdata = new MemSourceStreamOp(Arrays.asList(vecrows), veccolNames);
    Pipeline pl = new Pipeline().add(softmax).add(vsoftmax).add(svsoftmax).add(vssoftmax);
    PipelineModel model = pl.fit(vecdata);
    BatchOperator<?> result = model.transform(vecdata).select(new String[] { "label", "predLr", "vpredLr", "svpredLr" });
    List<Row> data = result.lazyPrint(100).collect();
    for (Row row : data) {
        for (int i = 1; i < 3; ++i) {
            Assert.assertEquals(row.getField(0), row.getField(i));
        }
    }
    // below is stream test code
    // below is stream test code.
    CollectSinkStreamOp sop = model.transform(svecdata).select(new String[] { "label", "predLr", "vpredLr", "svpredLr" }).link(new CollectSinkStreamOp());
    StreamOperator.execute();
    List<Row> rows = sop.getAndRemoveValues();
    for (Row row : rows) {
        for (int i = 1; i < 3; ++i) {
            Assert.assertEquals(row.getField(0), row.getField(i));
        }
    }
}

Example 2 with CollectSinkStreamOp

use of com.alibaba.alink.operator.stream.sink.CollectSinkStreamOp in project Alink by alibaba.

the class FlattenMTableStreamTest method linkFrom.

@Test
public void linkFrom() throws Exception {
    List<Row> rows = new ArrayList<>();
    rows.add(Row.of(1, "2", 0, null, new SparseVector(3, new int[] { 1 }, new double[] { 2.0 }), new FloatTensor(new float[] { 3.0f })));
    rows.add(Row.of(null, "2", 0, new DenseVector(new double[] { 0.0, 1.0 }), new SparseVector(4, new int[] { 2 }, new double[] { 3.0 }), new FloatTensor(new float[] { 3.0f })));
    rows.add(Row.of(null, "2", 0, new DenseVector(new double[] { 0.1, 1.0 }), new SparseVector(4, new int[] { 2 }, new double[] { 3.0 }), new FloatTensor(new float[] { 3.0f })));
    String schemaStr = "col0 int, col1 string, label int" + ", d_vec DENSE_VECTOR" + ", s_vec SPARSE_VECTOR" + ", tensor FLOAT_TENSOR";
    MTable mTable = new MTable(rows, schemaStr);
    List<Row> table = new ArrayList<>();
    table.add(Row.of("id", mTable.toString()));
    StreamOperator<?> op = new MemSourceStreamOp(table, new String[] { "id", "mTable" });
    StreamOperator<?> res = op.link(new FlattenMTableStreamOp().setSchemaStr(schemaStr).setSelectedCol("mTable").setReservedCols("id"));
    CollectSinkStreamOp sop = res.link(new CollectSinkStreamOp());
    StreamOperator.execute();
    List<Row> list = sop.getAndRemoveValues();
    for (Row row : list) {
        Assert.assertEquals(row.getField(0), "id");
    }
}

Example 3 with CollectSinkStreamOp

use of com.alibaba.alink.operator.stream.sink.CollectSinkStreamOp in project Alink by alibaba.

the class LookupModelStreamOpTest method testLookUpModelStream.

@Test
public void testLookUpModelStream() throws Exception {
    Row[] modelRows = new Row[] { Row.of(true, "ppk1", "res1", "rv1"), Row.of(true, "ppk9", "res1", "rv11"), Row.of(true, "ppk1", "res1", "rv111"), Row.of(true, "ppk2", "rsk2", "rv2"), Row.of(true, "ppk2", "rsk2", "rv22"), Row.of(true, "ppk2", "rsk222", "rv222"), Row.of(true, "ppk3", "rsk3", "rv3"), Row.of(true, "ppk5", "rsk5", "rv5"), Row.of(true, "ppk5", "rsk555", "rv555") };
    Row[] streamRows = new Row[] { Row.of(new Timestamp(0L), 3L, true, "ppk1", "res1", "rv1"), Row.of(new Timestamp(0L), 3L, true, "ppk2", "res1", "rv11"), Row.of(new Timestamp(0L), 3L, true, "ppk4", "res1", "rv2222"), Row.of(new Timestamp(1L), 3L, true, "ppk4", "rsk2", "rv2"), Row.of(new Timestamp(1L), 3L, true, "ppk2", "rsk2", "rv22"), Row.of(new Timestamp(1L), 3L, true, "ppk2", "rsk222", "rv222"), Row.of(new Timestamp(2L), 3L, true, "ppk3", "rsk3", "rv3"), Row.of(new Timestamp(2L), 3L, true, "ppk5", "rsk5", "rv5"), Row.of(new Timestamp(2L), 3L, true, "ppk5", "rsk555", "rv555") };
    BatchOperator<?> model = new MemSourceBatchOp(modelRows, new String[] { "_model_update_type_", "d", "e", "f" });
    StreamOperator<?> left = new MemSourceStreamOp(streamRows, new String[] { "alinkmodelstreamtimestamp", "alinkmodelstreamcount", "_model_update_type_", "d", "e", "f" });
    StreamOperator<?> data = new FakeStreamOperator(null);
    StreamOperator<?> out = new LookupStreamOp(model).setSelectedCols(new String[] { "a", "b" }).setMapKeyCols(new String[] { "d", "e" }).setMapValueCols("f").linkFrom(data, left);
    CollectSinkStreamOp sop = out.link(new CollectSinkStreamOp());
    StreamOperator.execute();
    List<Row> res = sop.getAndRemoveValues();
    Assert.assertEquals(30, res.size());
}

Example 4 with CollectSinkStreamOp

use of com.alibaba.alink.operator.stream.sink.CollectSinkStreamOp in project Alink by alibaba.

the class VectorToTensorStreamOpTest method testVectorToTensorStreamOp.

@Test
public void testVectorToTensorStreamOp() throws Exception {
    List<Row> data = Collections.singletonList(Row.of("0.0 0.1 1.0 1.1 2.0 2.1"));
    MemSourceStreamOp memSourceStreamOp = new MemSourceStreamOp(data, "vec string");
    CollectSinkStreamOp collectSinkStreamOp = memSourceStreamOp.link(new VectorToTensorStreamOp().setSelectedCol("vec")).link(new CollectSinkStreamOp());
    StreamOperator.execute();
    List<Row> ret = collectSinkStreamOp.getAndRemoveValues();
    Assert.assertEquals(1, ret.size());
    Assert.assertTrue(ret.get(0).getField(0) instanceof DoubleTensor);
    Assert.assertEquals(TensorUtil.getTensor("DOUBLE#6#0.0 0.1 1.0 1.1 2.0 2.1"), ret.get(0).getField(0));
}

Example 5 with CollectSinkStreamOp

use of com.alibaba.alink.operator.stream.sink.CollectSinkStreamOp in project Alink by alibaba.

the class VectorBiFunctionStreamOpTest method testBiVectorFunctionSparse.

@Test
public void testBiVectorFunctionSparse() throws Exception {
    List<Row> df = new ArrayList<>();
    df.add(Row.of("$8$1:3,2:4,4:7", "$8$1:3,2:4,4:7"));
    StreamOperator<?> data = new MemSourceStreamOp(df, "vec1 string, vec2 string");
    VectorBiFunctionStreamOp vecBiFunc = new VectorBiFunctionStreamOp().setSelectedCols("vec1", "vec2").setBiFuncName("minus").setReservedCols(new String[] {}).setOutputCol("vec_minus");
    StreamOperator<?> res = vecBiFunc.linkFrom(data);
    CollectSinkStreamOp sop = res.link(new CollectSinkStreamOp());
    StreamOperator.execute();
    List<Row> list = sop.getAndRemoveValues();
    for (Row row : list) {
        Assert.assertEquals(row.getField(0), VectorUtil.parseSparse("$8$1:0.0 2:0.0 4:0.0"));
    }
}