Examples with Configuration org.apache.inlong.sort.configuration.Configuration used on opensource projects

Example 1 with Configuration

use of org.apache.inlong.sort.configuration.Configuration in project incubator-inlong by apache.

the class ZkTools method updateDataFlowInfo.

/**
 * Update DataFlowInfo.
 * @param dataFlowInfo DataFlowInfo to be updated to.
 * @param cluster cluster name of the etl job.
 * @param dataFlowId dataFlow id
 * @param zkQuorum zk quorum
 * @param zkRoot zk root path
 * @throws Exception
 */
public static void updateDataFlowInfo(DataFlowInfo dataFlowInfo, String cluster, long dataFlowId, String zkQuorum, String zkRoot) throws Exception {
    final Configuration config = new Configuration();
    config.setString(Constants.ZOOKEEPER_QUORUM, zkQuorum);
    config.setString(Constants.ZOOKEEPER_ROOT, zkRoot);
    final ObjectMapper objectMapper = new ObjectMapper();
    final String dataFlowInfoPath = getNodePathOfDataFlowInfo(dataFlowId);
    final String dataFlowStorageInfoPath = getNodePathOfDataFlowStorageInfoInCluster(cluster, dataFlowId);
    try (CuratorFramework zkClient = ZooKeeperUtils.startCuratorFramework(config)) {
        // update DataFlowInfo
        final byte[] dataFlowInfoData = objectMapper.writeValueAsBytes(dataFlowInfo);
        if (zkClient.checkExists().forPath(dataFlowInfoPath) == null) {
            ZooKeeperUtils.createRecursive(zkClient, dataFlowInfoPath, dataFlowInfoData, CreateMode.PERSISTENT);
        } else {
            zkClient.setData().forPath(dataFlowInfoPath, dataFlowInfoData);
        }
        // update DataFlowStorageInfo
        DataFlowStorageInfo dataFlowStorageInfo = new DataFlowStorageInfo(ZK, dataFlowInfoPath);
        final byte[] dataFlowStorageInfoData = objectMapper.writeValueAsBytes(dataFlowStorageInfo);
        if (zkClient.checkExists().forPath(dataFlowStorageInfoPath) == null) {
            ZooKeeperUtils.createRecursive(zkClient, dataFlowStorageInfoPath, dataFlowStorageInfoData, CreateMode.PERSISTENT);
        } else {
            zkClient.setData().forPath(dataFlowStorageInfoPath, dataFlowStorageInfoData);
        }
    }
    LOG.info("Update DataFlowInfo with id {} on zk successfully", dataFlowId);
}

Example 2 with Configuration

use of org.apache.inlong.sort.configuration.Configuration in project incubator-inlong by apache.

the class CommonUtils method mergeConfAndProp.

public static Configuration mergeConfAndProp(Configuration conf, Map<String, Object> props) {
    final Configuration newConf = new Configuration(conf);
    newConf.addAll(props);
    return newConf;
}

Example 3 with Configuration

use of org.apache.inlong.sort.configuration.Configuration in project incubator-inlong by apache.

the class MultiTopicTubeSourceFunctionTest method testCommitTubeSubscriptionDescriptionPreview.

@Test
public void testCommitTubeSubscriptionDescriptionPreview() {
    final TestingMultiTenancyTubeConsumer tubeConsumer = generateTubeConsumer();
    final MultiTopicTubeSourceFunction sourceFunction = new MultiTopicTubeSourceFunction(new Configuration());
    sourceFunction.setTubeConsumer(tubeConsumer);
    sourceFunction.setTubeTopicSubscriptions(Collections.emptyMap());
    final Map<String, TubeSubscriptionDescription> descriptionPreview = new HashMap<>();
    descriptionPreview.put(topicAdded, TubeSubscriptionDescription.generate(dataFlowId1, generateTubeSourceInfo(topicAdded, tid1)));
    sourceFunction.commitTubeSubscriptionDescriptionPreview(descriptionPreview);
    assertEquals(sourceFunction.getTubeTopicSubscriptions(), descriptionPreview);
    assertEquals(1, tubeConsumer.addedTopics.size());
    assertEquals(topicAdded, tubeConsumer.addedTopics.get(0).getTopic());
    assertEquals(masterAddress, tubeConsumer.addedTopics.get(0).getMasterAddress());
    assertNull(tubeConsumer.addedTopics.get(0).getConsumerGroup());
    assertEquals(Collections.singletonList(tid1), tubeConsumer.addedTopics.get(0).getTids());
}

Example 4 with Configuration

use of org.apache.inlong.sort.configuration.Configuration in project incubator-inlong by apache.

the class MultiTopicTubeSourceFunctionTest method testProcessSourceEvent.

@Test
public void testProcessSourceEvent() {
    final TestingMultiTenancyTubeConsumer tubeConsumer = generateTubeConsumer();
    final MultiTopicTubeSourceFunction sourceFunction = new MultiTopicTubeSourceFunction(new Configuration());
    sourceFunction.setTubeConsumer(tubeConsumer);
    final Map<String, TubeSubscriptionDescription> descriptions = new HashMap<>();
    // add event1
    final SourceEvent addEvent1 = new SourceEvent(SourceEventType.ADDED, dataFlowId1, generateTubeSourceInfo(topicAdded, tid1));
    sourceFunction.processSourceEvent(addEvent1, descriptions);
    // add event should only change the preview
    assertTrue(tubeConsumer.addedTopics.isEmpty());
    assertTrue(tubeConsumer.updatedTopics.isEmpty());
    assertTrue(tubeConsumer.removedTopics.isEmpty());
    // add event1 again, should affect nothing
    sourceFunction.processSourceEvent(addEvent1, descriptions);
    // add event2
    final SourceEvent addEvent2 = new SourceEvent(SourceEventType.ADDED, dataFlowId2, generateTubeSourceInfo(topicAdded, tid2));
    sourceFunction.processSourceEvent(addEvent2, descriptions);
    // add event should only change the preview
    assertTrue(tubeConsumer.addedTopics.isEmpty());
    assertTrue(tubeConsumer.updatedTopics.isEmpty());
    assertTrue(tubeConsumer.removedTopics.isEmpty());
    assertEquals(1, descriptions.size());
    assertEquals(2, descriptions.get(topicAdded).getDataFlows().size());
    // remove event
    final TubeSubscriptionDescription existingDescription = new TubeSubscriptionDescription(topicRemoved, masterAddress, null);
    existingDescription.addDataFlow(dataFlowId3, generateTubeSourceInfo(topicRemoved, tid1));
    descriptions.put(topicRemoved, existingDescription);
    final SourceEvent removeEvent = new SourceEvent(SourceEventType.REMOVED, dataFlowId3, generateTubeSourceInfo(topicRemoved, tid1));
    sourceFunction.processSourceEvent(removeEvent, descriptions);
    assertTrue(tubeConsumer.addedTopics.isEmpty());
    assertTrue(tubeConsumer.updatedTopics.isEmpty());
    // remove event should be fired immediately
    assertEquals(1, tubeConsumer.removedTopics.size());
    assertEquals(topicRemoved, tubeConsumer.removedTopics.get(0));
    assertEquals(1, descriptions.size());
    assertEquals(2, descriptions.get(topicAdded).getDataFlows().size());
}

Example 5 with Configuration

use of org.apache.inlong.sort.configuration.Configuration in project incubator-inlong by apache.

the class MultiTopicTubeSourceFunctionTest method generateTubeConsumer.

private TestingMultiTenancyTubeConsumer generateTubeConsumer() {
    final Configuration configuration = new Configuration();
    configuration.setString(Constants.CLUSTER_ID, "cluster_id");
    return new TestingMultiTenancyTubeConsumer(configuration);
}