Examples with RawData com.navercorp.pinpoint.flink.vo.RawData used on opensource projects

Example 1 with RawData

use of com.navercorp.pinpoint.flink.vo.RawData in project pinpoint by naver.

the class AgentStatHandler method handleSimple.

@Override
public void handleSimple(ServerRequest<TBase<?, ?>> serverRequest) {
    final TBase<?, ?> tBase = serverRequest.getData();
    final Map<String, String> metaInfo = new HashMap<>(serverRequest.getHeaderEntity().getEntityAll());
    final RawData rawData = new RawData(tBase, metaInfo);
    final SourceContext sourceContext = roundRobinSourceContext();
    if (sourceContext == null) {
        logger.warn("sourceContext is null.");
        return;
    }
    sourceContext.collect(rawData);
}

Example 2 with RawData

use of com.navercorp.pinpoint.flink.vo.RawData in project pinpoint by naver.

the class TBaseFlatMapperTest method flatMap5Test.

@Test
public void flatMap5Test() throws Exception {
    ApplicationCache applicationCache = newMockApplicationCache();
    TBaseFlatMapper mapper = new TBaseFlatMapper(new JoinAgentStatBoMapper(), applicationCache, new DefaultTBaseFlatMapperInterceptor());
    TFAgentStatBatch tfAgentStatBatch = createTFAgentStatBatch5();
    ArrayList<Tuple3<String, JoinStatBo, Long>> dataList = new ArrayList<>();
    ListCollector<Tuple3<String, JoinStatBo, Long>> collector = new ListCollector<>(dataList);
    RawData rawData = newRawData(tfAgentStatBatch);
    mapper.flatMap(rawData, collector);
    assertEquals(dataList.size(), 2);
    Tuple3<String, JoinStatBo, Long> data1 = dataList.get(0);
    assertEquals(data1.f0, AGENT_ID);
    assertEquals(data1.f2.longValue(), 1491274143454L);
    JoinAgentStatBo joinAgentStatBo = (JoinAgentStatBo) data1.f1;
    assertEquals(joinAgentStatBo.getId(), AGENT_ID);
    assertEquals(joinAgentStatBo.getAgentStartTimestamp(), 1491274142454L);
    assertEquals(joinAgentStatBo.getTimestamp(), 1491274143454L);
    assertJoinDirectBufferBo(joinAgentStatBo.getJoinDirectBufferBoList());
    Tuple3<String, JoinStatBo, Long> data2 = dataList.get(1);
    assertEquals(data2.f0, APPLICATION_ID);
    assertEquals(data2.f2.longValue(), 1491274140000L);
    JoinApplicationStatBo joinApplicationStatBo = (JoinApplicationStatBo) data2.f1;
    assertEquals(joinApplicationStatBo.getId(), APPLICATION_ID);
    assertEquals(joinApplicationStatBo.getTimestamp(), 1491274140000L);
    assertEquals(joinApplicationStatBo.getStatType(), StatType.APP_STST);
    assertJoinDirectBufferBo(joinApplicationStatBo.getJoinDirectBufferBoList());
}

Example 3 with RawData

use of com.navercorp.pinpoint.flink.vo.RawData in project pinpoint by naver.

the class TBaseFlatMapperTest method flatMap4Test.

@Test
public void flatMap4Test() throws Exception {
    ApplicationCache applicationCache = newMockApplicationCache();
    TBaseFlatMapper mapper = new TBaseFlatMapper(new JoinAgentStatBoMapper(), applicationCache, new DefaultTBaseFlatMapperInterceptor());
    TFAgentStatBatch tfAgentStatBatch = createTFAgentStatBatch4();
    ArrayList<Tuple3<String, JoinStatBo, Long>> dataList = new ArrayList<>();
    ListCollector<Tuple3<String, JoinStatBo, Long>> collector = new ListCollector<>(dataList);
    RawData rawData = newRawData(tfAgentStatBatch);
    mapper.flatMap(rawData, collector);
    assertEquals(dataList.size(), 2);
    Tuple3<String, JoinStatBo, Long> data1 = dataList.get(0);
    assertEquals(data1.f0, AGENT_ID);
    assertEquals(data1.f2.longValue(), 1491274143454L);
    JoinAgentStatBo joinAgentStatBo = (JoinAgentStatBo) data1.f1;
    assertEquals(joinAgentStatBo.getId(), AGENT_ID);
    assertEquals(joinAgentStatBo.getAgentStartTimestamp(), 1491274142454L);
    assertEquals(joinAgentStatBo.getTimestamp(), 1491274143454L);
    assertJoinFileDescriptorBo(joinAgentStatBo.getJoinFileDescriptorBoList());
    Tuple3<String, JoinStatBo, Long> data2 = dataList.get(1);
    assertEquals(data2.f0, APPLICATION_ID);
    assertEquals(data2.f2.longValue(), 1491274140000L);
    JoinApplicationStatBo joinApplicationStatBo = (JoinApplicationStatBo) data2.f1;
    assertEquals(joinApplicationStatBo.getId(), APPLICATION_ID);
    assertEquals(joinApplicationStatBo.getTimestamp(), 1491274140000L);
    assertEquals(joinApplicationStatBo.getStatType(), StatType.APP_STST);
    assertJoinFileDescriptorBo(joinApplicationStatBo.getJoinFileDescriptorBoList());
}

Example 4 with RawData

use of com.navercorp.pinpoint.flink.vo.RawData in project pinpoint by naver.

the class TBaseFlatMapperTest method flatMap3Test.

@Test
public void flatMap3Test() throws Exception {
    ApplicationCache applicationCache = newMockApplicationCache();
    TBaseFlatMapper mapper = new TBaseFlatMapper(new JoinAgentStatBoMapper(), applicationCache, new DefaultTBaseFlatMapperInterceptor());
    TFAgentStatBatch tfAgentStatBatch = createTFAgentStatBatch3();
    ArrayList<Tuple3<String, JoinStatBo, Long>> dataList = new ArrayList<>();
    ListCollector<Tuple3<String, JoinStatBo, Long>> collector = new ListCollector<>(dataList);
    RawData rawData = newRawData(tfAgentStatBatch);
    mapper.flatMap(rawData, collector);
    assertEquals(dataList.size(), 2);
    Tuple3<String, JoinStatBo, Long> data1 = dataList.get(0);
    assertEquals(data1.f0, AGENT_ID);
    assertEquals(data1.f2.longValue(), 1491274143454L);
    JoinAgentStatBo joinAgentStatBo = (JoinAgentStatBo) data1.f1;
    assertEquals(joinAgentStatBo.getId(), AGENT_ID);
    assertEquals(joinAgentStatBo.getAgentStartTimestamp(), 1491274142454L);
    assertEquals(joinAgentStatBo.getTimestamp(), 1491274143454L);
    assertJoinTransactionBo(joinAgentStatBo.getJoinTransactionBoList());
    Tuple3<String, JoinStatBo, Long> data2 = dataList.get(1);
    assertEquals(data2.f0, APPLICATION_ID);
    assertEquals(data2.f2.longValue(), 1491274140000L);
    JoinApplicationStatBo joinApplicationStatBo = (JoinApplicationStatBo) data2.f1;
    assertEquals(joinApplicationStatBo.getId(), APPLICATION_ID);
    assertEquals(joinApplicationStatBo.getTimestamp(), 1491274140000L);
    assertEquals(joinApplicationStatBo.getStatType(), StatType.APP_STST);
    assertJoinTransactionBo(joinApplicationStatBo.getJoinTransactionBoList());
}

Example 5 with RawData

use of com.navercorp.pinpoint.flink.vo.RawData in project pinpoint by naver.

the class StatStreamingVer2Job method start.

public void start(ParameterTool parameters) throws Exception {
    logger.info("start Aggregation Stat Data job with job parameter. : " + parameters.toMap());
    final Bootstrap bootstrap = Bootstrap.getInstance(parameters.toMap());
    // set data source
    final TcpSourceFunction tcpSourceFunction = bootstrap.getTcpSourceFunction();
    final StreamExecutionEnvironment env = bootstrap.createStreamExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(parameters);
    DataStreamSource<RawData> rawData = env.addSource(tcpSourceFunction);
    // 0. generation rawdata
    final SingleOutputStreamOperator<Tuple3<String, JoinStatBo, Long>> statOperator = rawData.flatMap(bootstrap.getTbaseFlatMapper());
    // 1-1 save data processing application stat raw data
    final StatisticsDao statisticsDao = bootstrap.getStatisticsDao();
    DataStream<Tuple3<String, JoinStatBo, Long>> applicationStatAggregationData = statOperator.filter(new ApplicationStatBoFilter()).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple3<String, JoinStatBo, Long>>forMonotonousTimestamps().withTimestampAssigner(new AgentStatTimestampAssigner())).keyBy(new ApplicationStatKeySelector()).window(TumblingEventTimeWindows.of(Time.milliseconds(ApplicationStatBoWindow.WINDOW_SIZE))).allowedLateness(Time.milliseconds(ApplicationStatBoWindow.ALLOWED_LATENESS)).apply(new ApplicationStatBoWindow());
    applicationStatAggregationData.addSink(statisticsDao);
    // 1-2. aggregate application stat data
    // statOperator.filter(new FilterFunction<Tuple3<String, JoinStatBo, Long>>() {
    // @Override
    // public boolean filter(Tuple3<String, JoinStatBo, Long> value) throws Exception {
    // if (value.f1 instanceof JoinApplicationStatBo) {
    // logger.info("1-2 application stat aggre window function : " + value.f1);
    // return true;
    // }
    // 
    // return false;
    // }
    // })
    // .assignTimestampsAndWatermarks(new Timestamp())
    // .keyBy(0)
    // .window(TumblingEventTimeWindows.of(Time.seconds(120)))
    // .apply(new WindowFunction<Tuple3<String, JoinStatBo, Long>, Tuple3<String, JoinStatBo, Long>, Tuple, TimeWindow>() {
    // @Override
    // public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple3<String, JoinStatBo, Long>> values, Collector<Tuple3<String, JoinStatBo, Long>> out) throws Exception {
    // try {
    // JoinApplicationStatBo joinApplicationStatBo = join(values);
    // logger.info("1-2 application stat aggre window function : " + joinApplicationStatBo);
    // out.collect(new Tuple3<>(joinApplicationStatBo.getId(), joinApplicationStatBo, joinApplicationStatBo.getTimestamp()));
    // } catch (Exception e) {
    // logger.error("window function error", e);
    // }
    // }
    // 
    // private JoinApplicationStatBo join(Iterable<Tuple3<String, JoinStatBo, Long>> values) {
    // List<JoinApplicationStatBo> joinApplicaitonStatBoList = new ArrayList<JoinApplicationStatBo>();
    // for (Tuple3<String, JoinStatBo, Long> value : values) {
    // joinApplicaitonStatBoList.add((JoinApplicationStatBo) value.f1);
    // }
    // return JoinApplicationStatBo.joinApplicationStatBo(joinApplicaitonStatBoList);
    // 
    // }
    // }).writeUsingOutputFormat(statisticsDao);
    // 2. agrregage agent stat
    // statOperator.filter(new FilterFunction<Tuple3<String, JoinStatBo, Long>>() {
    // @Override
    // public boolean filter(Tuple3<String, JoinStatBo, Long> value) throws Exception {
    // if (value.f1 instanceof JoinAgentStatBo) {
    // logger.info("2 application stat aggre window function : " + value.f1);
    // return true;
    // }
    // 
    // return false;
    // }
    // })
    // .assignTimestampsAndWatermarks(new Timestamp())
    // .keyBy(0)
    // .window(TumblingEventTimeWindows.of(Time.seconds(120)))
    // 
    // .apply(new WindowFunction<Tuple3<String, JoinStatBo, Long>, Tuple3<String, JoinStatBo, Long>, Tuple, TimeWindow>() {
    // 
    // @Override
    // public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple3<String, JoinStatBo, Long>> values, Collector<Tuple3<String, JoinStatBo, Long>> out) throws Exception {
    // try {
    // JoinAgentStatBo joinAgentStatBo = join(values);
    // logger.info("2 agent stat aggre window function : " + joinAgentStatBo);
    // out.collect(new Tuple3<>(joinAgentStatBo.getId(), joinAgentStatBo, joinAgentStatBo.getTimestamp()));
    // } catch (Exception e) {
    // logger.error("window function error", e);
    // }
    // }
    // 
    // private JoinAgentStatBo join(Iterable<Tuple3<String, JoinStatBo, Long>> values) {
    // List<JoinAgentStatBo> joinAgentStatBoList =  new ArrayList<JoinAgentStatBo>();
    // for (Tuple3<String, JoinStatBo, Long> value : values) {
    // joinAgentStatBoList.add((JoinAgentStatBo) value.f1);
    // }
    // 
    // return JoinAgentStatBo.joinAgentStatBo(joinAgentStatBoList);
    // }
    // })
    // .writeUsingOutputFormat(statisticsDao);
    env.execute("Aggregation Stat Data");
}