use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.
the class CanalFilterSupport method makeFilterExpression.
/**
* 构建filter 表达式
*/
public static String makeFilterExpression(Pipeline pipeline) {
List<DataMediaPair> dataMediaPairs = pipeline.getPairs();
if (dataMediaPairs.isEmpty()) {
throw new SelectException("ERROR ## the pair is empty,the pipeline id = " + pipeline.getId());
}
Set<String> mediaNames = new HashSet<String>();
for (DataMediaPair dataMediaPair : dataMediaPairs) {
DataMedia.ModeValue namespaceMode = dataMediaPair.getSource().getNamespaceMode();
DataMedia.ModeValue nameMode = dataMediaPair.getSource().getNameMode();
if (namespaceMode.getMode().isSingle()) {
buildFilter(mediaNames, namespaceMode.getSingleValue(), nameMode, false);
} else if (namespaceMode.getMode().isMulti()) {
for (String namespace : namespaceMode.getMultiValue()) {
buildFilter(mediaNames, namespace, nameMode, false);
}
} else if (namespaceMode.getMode().isWildCard()) {
buildFilter(mediaNames, namespaceMode.getSingleValue(), nameMode, true);
}
}
StringBuilder result = new StringBuilder();
Iterator<String> iter = mediaNames.iterator();
int i = -1;
while (iter.hasNext()) {
i++;
if (i == 0) {
result.append(iter.next());
} else {
result.append(",").append(iter.next());
}
}
String markTable = pipeline.getParameters().getSystemSchema() + "." + pipeline.getParameters().getSystemMarkTable();
String bufferTable = pipeline.getParameters().getSystemSchema() + "." + pipeline.getParameters().getSystemBufferTable();
String dualTable = pipeline.getParameters().getSystemSchema() + "." + pipeline.getParameters().getSystemDualTable();
if (!mediaNames.contains(markTable)) {
result.append(",").append(markTable);
}
if (!mediaNames.contains(bufferTable)) {
result.append(",").append(bufferTable);
}
if (!mediaNames.contains(dualTable)) {
result.append(",").append(dualTable);
}
return result.toString();
}
use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.
the class LocalFileLoaderActionTest method test_load_file.
@Test
public void test_load_file() {
final Pipeline pipeline = new Pipeline();
pipeline.setId(100L);
List<DataMediaPair> pairs = generatorDataMediaPair(10);
pipeline.setPairs(pairs);
new NonStrictExpectations() {
{
configClientService.findPipeline(anyLong);
returns(pipeline);
}
};
Identity identity = new Identity();
identity.setChannelId(100L);
identity.setPipelineId(100L);
identity.setProcessId(100L);
FileBatch fileBatch = new FileBatch();
fileBatch.setIdentity(identity);
fileBatch.getFiles().addAll(generatorLocalFileData("fileLoad", 10));
WeightController controller = new WeightController(1);
fileLoadAction.load(fileBatch, new File(tmp + File.separator + OTTERLOAD), controller);
File target = new File(tmp + File.separator + OTTERLOAD + "_loaded/");
want.number(target.listFiles().length).isEqualTo(10);
NioUtils.delete(target);
}
use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.
the class OtterTransformerTest method test_rowData_oracle_mysql.
@Test
public void test_rowData_oracle_mysql() {
final Pipeline pipeline = new Pipeline();
pipeline.setId(100L);
List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
DataMediaPair pair1 = new DataMediaPair();
pair1.setId(1L);
pair1.setPipelineId(pipeline.getId());
pair1.setPullWeight(1L);
pair1.setPushWeight(1L);
DbDataMedia oracleMedia = getOracleMedia();
oracleMedia.setId(1L);
pair1.setSource(oracleMedia);
DbDataMedia mysqlMedia = getMysqlMedia();
pair1.setTarget(mysqlMedia);
pairs.add(pair1);
pipeline.setPairs(pairs);
PipelineParameter param = new PipelineParameter();
param.setSyncMode(SyncMode.ROW);
pipeline.setParameters(param);
new NonStrictExpectations() {
{
configClientService.findPipeline(anyLong);
returns(pipeline);
}
};
Identity identity = new Identity();
identity.setChannelId(100L);
identity.setPipelineId(100L);
identity.setProcessId(100L);
RowBatch rowBatch = new RowBatch();
rowBatch.setIdentity(identity);
EventData eventData = new EventData();
eventData.setTableId(1L);
eventData.setSchemaName("srf");
eventData.setTableName("columns");
eventData.setEventType(EventType.UPDATE);
eventData.setExecuteTime(100L);
eventData.getKeys().add(buildColumn("id", Types.NUMERIC, "1", true, false));
eventData.getKeys().add(buildColumn("name", Types.VARCHAR, "ljh", true, false));
eventData.getColumns().add(buildColumn("alias_name", Types.CHAR, "hello", false, false));
eventData.getColumns().add(buildColumn("amount", Types.NUMERIC, "100.01", false, false));
eventData.getColumns().add(buildColumn("text_b", Types.BLOB, "[116,101,120,116,95,98]", false, false));
eventData.getColumns().add(buildColumn("text_c", Types.CLOB, "text_c", false, false));
eventData.getColumns().add(buildColumn("curr_date", Types.DATE, "2011-01-01", false, false));
eventData.getColumns().add(buildColumn("gmt_create", Types.DATE, "2011-01-01 11:11:11", false, false));
eventData.getColumns().add(buildColumn("gmt_modify", Types.DATE, "2011-01-01 11:11:11", false, false));
rowBatch.merge(eventData);
Map<Class, BatchObject> batchs = otterTransformFactory.transform(rowBatch);
RowBatch result = (RowBatch) batchs.get(EventData.class);
want.number(result.getDatas().size()).isEqualTo(1);
}
use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.
the class DatabaseExtractorTest method getDataMediaPairForMysql.
private List<DataMediaPair> getDataMediaPairForMysql(long tableId, int count) {
List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
for (int i = 0; i < count; i++) {
DataMediaPair pair = new DataMediaPair();
pair.setId(Long.valueOf(i));
pair.setPullWeight(1L);
pair.setPushWeight(1L);
DbDataMedia mysqlMedia = getMysqlMedia();
mysqlMedia.setId(tableId + i);
pair.setSource(mysqlMedia);
DbDataMedia oracleMedia = getOracleMedia();
oracleMedia.setId(tableId + i + count);
pair.setTarget(oracleMedia);
pairs.add(pair);
}
return pairs;
}
use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.
the class DatabaseExtractorTest method test_override_field.
public void test_override_field() {
final Pipeline pipeline = new Pipeline();
pipeline.setId(100L);
pipeline.getParameters().setSyncMode(SyncMode.FIELD);
// 设置为全局
pipeline.getParameters().setSyncConsistency(SyncConsistency.BASE);
int start = RandomUtils.nextInt();
int count = 10;
List<DataMediaPair> pairs = getDataMediaPairForOracle(start, count);
pipeline.setPairs(pairs);
new NonStrictExpectations() {
{
configClientService.findPipeline(100L);
returns(pipeline);
}
};
// 构造数据
RowBatch rowBatch = new RowBatch();
rowBatch.setIdentity(identity);
for (int tableId = start; tableId < start + count; tableId++) {
for (int i = start; i < start + count; i++) {
EventData eventData = getEventData(tableId, i);
eventData.setSchemaName("srf");
eventData.setTableName("columns");
eventData.setSyncConsistency(SyncConsistency.MEDIA);
rowBatch.merge(eventData);
}
}
databaseExtractor.extract(new DbBatch(rowBatch));
want.number(rowBatch.getDatas().size()).isEqualTo(count);
}
Aggregations