Examples with SourceDataEntry io.openmessaging.connector.api.data.SourceDataEntry used on opensource projects

Example 6 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class KafkaSourceTaskTest method pollTest.

@Test
public void pollTest() throws Exception {
    KafkaSourceTask task = new KafkaSourceTask();
    Field config = KafkaSourceTask.class.getDeclaredField("config");
    config.setAccessible(true);
    Collection<SourceDataEntry> list = task.poll();
    Assert.assertEquals(list.size(), 1);
    list = task.poll();
    Assert.assertEquals(list.size(), 0);
}

Example 7 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class MongoDataEntry method createSouceDataEntry.

public static SourceDataEntry createSouceDataEntry(ReplicationEvent event, ReplicaSetConfig replicaSetConfig) {
    DataEntryBuilder dataEntryBuilder;
    if (event.getOperationType().equals(OperationType.CREATED)) {
        Schema schema = createdSchema(replicaSetConfig.getReplicaSetName());
        dataEntryBuilder = new DataEntryBuilder(schema);
        dataEntryBuilder.timestamp(System.currentTimeMillis()).queue(event.getNamespace().replace(".", "-").replace("$", "-")).entryType(event.getEntryType());
        dataEntryBuilder.putFiled(CREATED, event.getDocument().toJson());
        dataEntryBuilder.putFiled(NAMESPACE, event.getNamespace());
    } else {
        Schema schema = oplogSchema(replicaSetConfig.getReplicaSetName());
        dataEntryBuilder = new DataEntryBuilder(schema);
        dataEntryBuilder.timestamp(System.currentTimeMillis()).queue(event.getNamespace().replace(".", "-").replace("$", "-")).entryType(event.getEntryType());
        dataEntryBuilder.putFiled(OPERATION_TYPE, event.getOperationType().name());
        dataEntryBuilder.putFiled(TIMESTAMP, event.getTimestamp().getValue());
        dataEntryBuilder.putFiled(VERSION, event.getV());
        dataEntryBuilder.putFiled(NAMESPACE, event.getNamespace());
        dataEntryBuilder.putFiled(PATCH, event.getEventData().isPresent() ? JSONObject.toJSONString(event.getEventData().get()) : "");
        dataEntryBuilder.putFiled(OBJECT_ID, event.getObjectId().isPresent() ? JSONObject.toJSONString(event.getObjectId().get()) : "");
    }
    String position = createPosition(event, replicaSetConfig);
    SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(ByteBuffer.wrap(replicaSetConfig.getReplicaSetName().getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(position.getBytes(StandardCharsets.UTF_8)));
    return sourceDataEntry;
}

Example 8 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class JdbcSourceTask method poll.

@Override
public Collection<SourceDataEntry> poll() {
    List<SourceDataEntry> res = new ArrayList<>();
    try {
        if (tableQueue.size() > 1)
            querier = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
        else
            querier = tableQueue.peek();
        Timer timer = new Timer();
        try {
            Thread.currentThread();
            // 毫秒
            Thread.sleep(1000);
        } catch (Exception e) {
            throw e;
        }
        querier.poll();
        for (Table dataRow : querier.getList()) {
            JSONObject jsonObject = new JSONObject();
            jsonObject.put("nextQuery", "database");
            jsonObject.put("nextPosition", "table");
            Schema schema = new Schema();
            schema.setDataSource(dataRow.getDatabase());
            schema.setName(dataRow.getName());
            schema.setFields(new ArrayList<>());
            for (int i = 0; i < dataRow.getColList().size(); i++) {
                String columnName = dataRow.getColList().get(i);
                String rawDataType = dataRow.getRawDataTypeList().get(i);
                Field field = new Field(i, columnName, ColumnParser.mapConnectorFieldType(rawDataType));
                schema.getFields().add(field);
            }
            DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
            dataEntryBuilder.timestamp(System.currentTimeMillis()).queue(dataRow.getName()).entryType(EntryType.UPDATE);
            for (int i = 0; i < dataRow.getColList().size(); i++) {
                Object[] value = new Object[2];
                value[0] = value[1] = dataRow.getParserList().get(i).getValue(dataRow.getDataList().get(i));
                dataEntryBuilder.putFiled(dataRow.getColList().get(i), JSONObject.toJSONString(value));
            }
            SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(ByteBuffer.wrap((ConstDefine.PREFIX + config.getDbUrl() + config.getDbPort()).getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8)));
            res.add(sourceDataEntry);
            log.debug("sourceDataEntry : {}", JSONObject.toJSONString(sourceDataEntry));
        }
    } catch (Exception e) {
        log.error("JDBC task poll error, current config:" + JSON.toJSONString(config), e);
    }
    log.debug("dataEntry poll successfully,{}", JSONObject.toJSONString(res));
    return res;
}

Example 9 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class RedisEntryConverterTest method test.

@Test
public void test() {
    KVEntry entry = getKVEntry();
    KVEntryConverter converter = new RedisEntryConverter();
    Collection<SourceDataEntry> res = converter.kVEntryToDataEntries(entry);
    Assert.assertNotNull(res);
    Assert.assertEquals(1, res.size());
    Assert.assertEquals("key", ((List<SourceDataEntry>) res).get(0).getPayload()[1]);
    Assert.assertEquals("value", ((List<SourceDataEntry>) res).get(0).getPayload()[2]);
    Assert.assertEquals("set", ((List<SourceDataEntry>) res).get(0).getPayload()[0]);
    Map<String, Object> params = (Map<String, Object>) ((List<SourceDataEntry>) res).get(0).getPayload()[3];
    Assert.assertEquals("replId", params.get(Options.REDIS_REPLID.name()));
}

Example 10 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class WorkerSourceTask method run.

/**
 * Start a source task, and send data entry to MQ cyclically.
 */
@Override
public void run() {
    try {
        producer.start();
        log.info("Source task producer start.");
        state.compareAndSet(WorkerTaskState.NEW, WorkerTaskState.PENDING);
        sourceTask.initialize(new SourceTaskContext() {

            @Override
            public PositionStorageReader positionStorageReader() {
                return positionStorageReader;
            }

            @Override
            public KeyValue configs() {
                return taskConfig;
            }
        });
        sourceTask.start(taskConfig);
        state.compareAndSet(WorkerTaskState.PENDING, WorkerTaskState.RUNNING);
        log.info("Source task start, config:{}", JSON.toJSONString(taskConfig));
        while (WorkerState.STARTED == workerState.get() && WorkerTaskState.RUNNING == state.get()) {
            try {
                Collection<SourceDataEntry> toSendEntries = sourceTask.poll();
                if (null != toSendEntries && toSendEntries.size() > 0) {
                    sendRecord(toSendEntries);
                }
            } catch (Exception e) {
                log.error("Source task runtime exception", e);
                state.set(WorkerTaskState.ERROR);
            }
        }
        sourceTask.stop();
        state.compareAndSet(WorkerTaskState.STOPPING, WorkerTaskState.STOPPED);
        log.info("Source task stop, config:{}", JSON.toJSONString(taskConfig));
    } catch (Exception e) {
        log.error("Run task failed.", e);
        state.set(WorkerTaskState.ERROR);
    } finally {
        if (producer != null) {
            producer.shutdown();
            log.info("Source task producer shutdown.");
        }
    }
}