Examples with InlongGroupExtInfo org.apache.inlong.manager.common.pojo.group.InlongGroupExtInfo used on opensource projects

Example 6 with InlongGroupExtInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupExtInfo in project incubator-inlong by apache.

the class InlongGroupTransfer method parsePulsarConf.

private static PulsarBaseConf parsePulsarConf(InlongGroupResponse groupResponse) {
    PulsarBaseConf pulsarBaseConf = new PulsarBaseConf();
    pulsarBaseConf.setNamespace(groupResponse.getMqResourceObj());
    InlongGroupPulsarInfo inlongGroupPulsarInfo = (InlongGroupPulsarInfo) groupResponse.getMqExtInfo();
    pulsarBaseConf.setAckQuorum(inlongGroupPulsarInfo.getAckQuorum());
    pulsarBaseConf.setWriteQuorum(inlongGroupPulsarInfo.getWriteQuorum());
    pulsarBaseConf.setEnsemble(inlongGroupPulsarInfo.getEnsemble());
    pulsarBaseConf.setTtl(inlongGroupPulsarInfo.getTtl());
    pulsarBaseConf.setTenant(inlongGroupPulsarInfo.getTenant());
    pulsarBaseConf.setRetentionTime(inlongGroupPulsarInfo.getRetentionTime());
    pulsarBaseConf.setRetentionSize(inlongGroupPulsarInfo.getRetentionSize());
    pulsarBaseConf.setRetentionSizeUnit(inlongGroupPulsarInfo.getRetentionSizeUnit());
    pulsarBaseConf.setRetentionTimeUnit(inlongGroupPulsarInfo.getRetentionTimeUnit());
    pulsarBaseConf.setEnableCreateResource(inlongGroupPulsarInfo.getEnableCreateResource() == 1 ? true : false);
    List<InlongGroupExtInfo> groupExtInfos = groupResponse.getExtList();
    for (InlongGroupExtInfo extInfo : groupExtInfos) {
        if (extInfo.getKeyName().equals(InlongGroupSettings.PULSAR_ADMIN_URL)) {
            pulsarBaseConf.setPulsarAdminUrl(extInfo.getKeyValue());
        }
        if (extInfo.getKeyName().equals(InlongGroupSettings.PULSAR_SERVICE_URL)) {
            pulsarBaseConf.setPulsarServiceUrl(extInfo.getKeyValue());
        }
    }
    return pulsarBaseConf;
}

Example 7 with InlongGroupExtInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupExtInfo in project incubator-inlong by apache.

the class CreateSortConfigListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws Exception {
    LOGGER.info("Create sort config for context={}", context);
    ProcessForm form = context.getProcessForm();
    if (form instanceof UpdateGroupProcessForm) {
        UpdateGroupProcessForm updateGroupProcessForm = (UpdateGroupProcessForm) form;
        OperateType operateType = updateGroupProcessForm.getOperateType();
        if (operateType == OperateType.SUSPEND || operateType == OperateType.DELETE) {
            return ListenerResult.success();
        }
    }
    InlongGroupInfo groupInfo = this.getGroupInfo(form);
    String groupId = groupInfo.getInlongGroupId();
    if (StringUtils.isEmpty(groupId)) {
        LOGGER.warn("GroupId is null for context={}", context);
        return ListenerResult.success();
    }
    List<SinkResponse> sinkResponseList = streamSinkService.listSink(groupId, null);
    if (CollectionUtils.isEmpty(sinkResponseList)) {
        LOGGER.warn("Sink not found by groupId={}", groupId);
        return ListenerResult.success();
    }
    Map<String, DataFlowInfo> dataFlowInfoMap = sinkResponseList.stream().map(sink -> {
        DataFlowInfo flowInfo = commonOperateService.createDataFlow(groupInfo, sink);
        return Pair.of(sink.getInlongStreamId(), flowInfo);
    }).collect(Collectors.toMap(Pair::getKey, Pair::getValue));
    String dataFlows = OBJECT_MAPPER.writeValueAsString(dataFlowInfoMap);
    InlongGroupExtInfo extInfo = new InlongGroupExtInfo();
    extInfo.setInlongGroupId(groupId);
    extInfo.setKeyName(InlongGroupSettings.DATA_FLOW);
    extInfo.setKeyValue(dataFlows);
    if (groupInfo.getExtList() == null) {
        groupInfo.setExtList(Lists.newArrayList());
    }
    upsertDataFlow(groupInfo, extInfo);
    return ListenerResult.success();
}

Example 8 with InlongGroupExtInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupExtInfo in project incubator-inlong by apache.

the class InlongGroupServiceTest method testSaveAndUpdateExt.

@Test
public void testSaveAndUpdateExt() {
    // check insert
    InlongGroupExtInfo groupExtInfo1 = new InlongGroupExtInfo();
    groupExtInfo1.setId(1);
    groupExtInfo1.setInlongGroupId(globalGroupId);
    groupExtInfo1.setKeyName("pulsar_url");
    groupExtInfo1.setKeyValue("http://127.0.0.1:8080");
    InlongGroupExtInfo groupExtInfo2 = new InlongGroupExtInfo();
    groupExtInfo2.setId(2);
    groupExtInfo2.setInlongGroupId(globalGroupId);
    groupExtInfo2.setKeyName("pulsar_secret");
    groupExtInfo2.setKeyValue("QWEASDZXC");
    List<InlongGroupExtInfo> groupExtInfoList = Arrays.asList(groupExtInfo1, groupExtInfo2);
    groupService.saveOrUpdateExt(globalGroupId, groupExtInfoList);
    List<InlongGroupExtEntity> extEntityList = groupExtMapper.selectByGroupId(globalGroupId);
    Assert.assertEquals(2, extEntityList.size());
    Assert.assertEquals("pulsar_url", extEntityList.get(0).getKeyName());
    Assert.assertEquals("http://127.0.0.1:8080", extEntityList.get(0).getKeyValue());
    // check update
    groupExtInfo1.setKeyValue("http://127.0.0.1:8081");
    groupService.saveOrUpdateExt(globalGroupId, groupExtInfoList);
    extEntityList = groupExtMapper.selectByGroupId(globalGroupId);
    Assert.assertEquals(2, extEntityList.size());
    Assert.assertEquals("http://127.0.0.1:8081", extEntityList.get(0).getKeyValue());
    groupExtInfo2.setKeyValue("qweasdzxc");
    groupService.saveOrUpdateExt(globalGroupId, groupExtInfoList);
    extEntityList = groupExtMapper.selectByGroupId(globalGroupId);
    Assert.assertEquals(2, extEntityList.size());
    Assert.assertEquals("qweasdzxc", extEntityList.get(1).getKeyValue());
}

Example 9 with InlongGroupExtInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupExtInfo in project incubator-inlong by apache.

the class InlongGroupServiceImpl method get.

@Override
public InlongGroupInfo get(String groupId) {
    LOGGER.debug("begin to get inlong group info by groupId={}", groupId);
    Preconditions.checkNotNull(groupId, Constant.GROUP_ID_IS_EMPTY);
    InlongGroupEntity entity = groupMapper.selectByGroupId(groupId);
    if (entity == null) {
        LOGGER.error("inlong group not found by groupId={}", groupId);
        throw new BusinessException(ErrorCodeEnum.GROUP_NOT_FOUND);
    }
    InlongGroupInfo groupInfo = CommonBeanUtils.copyProperties(entity, InlongGroupInfo::new);
    List<InlongGroupExtEntity> extEntityList = groupExtMapper.selectByGroupId(groupId);
    List<InlongGroupExtInfo> extInfoList = CommonBeanUtils.copyListProperties(extEntityList, InlongGroupExtInfo::new);
    groupInfo.setExtList(extInfoList);
    // If the middleware is Pulsar, we need to encapsulate Pulsar related data
    String mqType = entity.getMiddlewareType();
    if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
        InlongGroupPulsarEntity pulsarEntity = groupPulsarMapper.selectByGroupId(groupId);
        Preconditions.checkNotNull(pulsarEntity, "Pulsar info not found by the groupId=" + groupId);
        InlongGroupPulsarInfo pulsarInfo = CommonBeanUtils.copyProperties(pulsarEntity, InlongGroupPulsarInfo::new);
        pulsarInfo.setMiddlewareType(mqType);
        groupInfo.setMqExtInfo(pulsarInfo);
    }
    // For approved inlong group, encapsulate the cluster address of the middleware
    if (GroupState.CONFIG_SUCCESSFUL == GroupState.forCode(groupInfo.getStatus())) {
        if (Constant.MIDDLEWARE_TUBE.equalsIgnoreCase(mqType)) {
            groupInfo.setTubeMaster(commonOperateService.getSpecifiedParam(Constant.TUBE_MASTER_URL));
        } else if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
            PulsarClusterInfo pulsarCluster = commonOperateService.getPulsarClusterInfo(mqType);
            groupInfo.setPulsarAdminUrl(pulsarCluster.getAdminUrl());
            groupInfo.setPulsarServiceUrl(pulsarCluster.getBrokerServiceUrl());
        }
    }
    LOGGER.debug("success to get inlong group for groupId={}", groupId);
    return groupInfo;
}

Example 10 with InlongGroupExtInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupExtInfo in project incubator-inlong by apache.

the class PulsarUtilsTest method testGetPulsarAdmin.

@Test
public void testGetPulsarAdmin() {
    InlongGroupExtInfo groupExtInfo1 = new InlongGroupExtInfo();
    groupExtInfo1.setId(1);
    groupExtInfo1.setInlongGroupId("group1");
    groupExtInfo1.setKeyName(InlongGroupSettings.PULSAR_ADMIN_URL);
    groupExtInfo1.setKeyValue("http://127.0.0.1:8080");
    InlongGroupExtInfo groupExtInfo2 = new InlongGroupExtInfo();
    groupExtInfo2.setId(2);
    groupExtInfo2.setInlongGroupId("group1");
    groupExtInfo2.setKeyName(InlongGroupSettings.PULSAR_AUTHENTICATION);
    groupExtInfo2.setKeyValue("QWEASDZXC");
    ArrayList<InlongGroupExtInfo> groupExtInfoList = Lists.newArrayList(groupExtInfo1, groupExtInfo2);
    InlongGroupInfo groupInfo = new InlongGroupInfo();
    groupInfo.setExtList(groupExtInfoList);
    final String defaultServiceUrl = "http://127.0.0.1:10080";
    try {
        PulsarAdmin admin = PulsarUtils.getPulsarAdmin(defaultServiceUrl);
        Assert.assertEquals("http://127.0.0.1:8080", admin.getServiceUrl());
        Field auth = ReflectionUtils.findField(PulsarAdminImpl.class, "auth");
        assert auth != null;
        auth.setAccessible(true);
        Authentication authentication = (Authentication) auth.get(admin);
        Assert.assertNotNull(authentication);
        InlongGroupExtInfo groupExtInfo3 = new InlongGroupExtInfo();
        groupExtInfo3.setId(3);
        groupExtInfo3.setInlongGroupId("group1");
        groupExtInfo3.setKeyName(InlongGroupSettings.PULSAR_AUTHENTICATION_TYPE);
        groupExtInfo3.setKeyValue("token1");
        groupExtInfoList.add(groupExtInfo3);
        try {
            admin = PulsarUtils.getPulsarAdmin(defaultServiceUrl);
        } catch (Exception e) {
            if (e instanceof IllegalArgumentException) {
                Assert.assertTrue(e.getMessage().contains("illegal authentication type"));
            }
        }
        groupExtInfoList = new ArrayList<>();
        groupInfo.setExtList(groupExtInfoList);
        admin = PulsarUtils.getPulsarAdmin(defaultServiceUrl);
        Assert.assertEquals("http://127.0.0.1:10080", admin.getServiceUrl());
        auth = ReflectionUtils.findField(PulsarAdminImpl.class, "auth");
        assert auth != null;
        auth.setAccessible(true);
        authentication = (Authentication) auth.get(admin);
        Assert.assertTrue(authentication instanceof AuthenticationDisabled);
    } catch (PulsarClientException | IllegalAccessException e) {
        Assert.fail();
    }
}