use of com.alibaba.otter.canal.parse.index.AbstractLogPositionManager in project canal by alibaba.
the class GroupEventPaserTest method buildEventParser.
private MysqlEventParser buildEventParser(int slaveId) {
MysqlEventParser mysqlEventPaser = new MysqlEventParser();
EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
mysqlEventPaser.setDestination("group-" + slaveId);
mysqlEventPaser.setSlaveId(slaveId);
mysqlEventPaser.setDetectingEnable(false);
mysqlEventPaser.setDetectingSQL(DETECTING_SQL);
mysqlEventPaser.setMasterInfo(buildAuthentication());
mysqlEventPaser.setMasterPosition(defaultPosition);
mysqlEventPaser.setBinlogParser(buildParser(buildAuthentication()));
mysqlEventPaser.setEventSink(new EntryEventSink());
mysqlEventPaser.setLogPositionManager(new AbstractLogPositionManager() {
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
System.out.println(logPosition);
}
});
return mysqlEventPaser;
}
use of com.alibaba.otter.canal.parse.index.AbstractLogPositionManager in project canal by alibaba.
the class LocalBinlogDumpTest method testSimple.
@Test
public void testSimple() {
String directory = "/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb";
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L);
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal"));
controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
controller.setDirectory(directory);
controller.setMasterPosition(startPosition);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
for (Entry entry : entrys) {
if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
continue;
}
if (entry.getEntryType() == EntryType.ROWDATA) {
RowChange rowChange = null;
try {
rowChange = RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
}
EventType eventType = rowChange.getEventType();
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
for (RowData rowData : rowChange.getRowDatasList()) {
if (eventType == EventType.DELETE) {
print(rowData.getBeforeColumnsList());
} else if (eventType == EventType.INSERT) {
print(rowData.getAfterColumnsList());
} else {
System.out.println("-------> before");
print(rowData.getBeforeColumnsList());
System.out.println("-------> after");
print(rowData.getAfterColumnsList());
}
}
}
}
return true;
}
});
controller.setLogPositionManager(new AbstractLogPositionManager() {
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
System.out.println(logPosition);
}
});
controller.start();
try {
Thread.sleep(100 * 1000L);
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
controller.stop();
}
use of com.alibaba.otter.canal.parse.index.AbstractLogPositionManager in project canal by alibaba.
the class RdsBinlogEventParserProxyTest method test_timestamp.
@Test
public void test_timestamp() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker(3000 * 1000);
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final RdsBinlogEventParserProxy controller = new RdsBinlogEventParserProxy();
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.HOUR_OF_DAY, -24 * 4);
final EntryPosition defaultPosition = buildPosition(null, null, calendar.getTimeInMillis());
controller.setSlaveId(3344L);
controller.setDetectingEnable(false);
controller.setDetectingSQL(DETECTING_SQL);
controller.setMasterInfo(buildAuthentication());
controller.setMasterPosition(defaultPosition);
controller.setInstanceId("");
controller.setAccesskey("");
controller.setSecretkey("");
controller.setDirectory("/tmp/binlog");
controller.setEventBlackFilter(new AviaterRegexFilter("mysql\\.*"));
controller.setFilterTableError(true);
controller.setBatchFileSize(4);
controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {
@Override
public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
for (CanalEntry.Entry entry : entrys) {
if (entry.getEntryType() != CanalEntry.EntryType.HEARTBEAT) {
entryCount.incrementAndGet();
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
}
return true;
}
});
controller.setLogPositionManager(new AbstractLogPositionManager() {
private LogPosition logPosition;
public void persistLogPosition(String destination, LogPosition logPosition) {
this.logPosition = logPosition;
}
public LogPosition getLatestIndexBy(String destination) {
return logPosition;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
Assert.assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
Assert.assertTrue(entryPosition.getPosition() <= 6163L);
Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}
use of com.alibaba.otter.canal.parse.index.AbstractLogPositionManager in project canal by alibaba.
the class LocalBinlogEventParserTest method test_position.
@Test
public void test_position() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker();
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", 219L, 1505467103000L);
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
controller.setMasterPosition(defaultPosition);
controller.setMasterInfo(buildAuthentication());
controller.setDirectory(directory);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
entryCount.incrementAndGet();
for (Entry entry : entrys) {
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
controller.stop();
timeoutChecker.stop();
timeoutChecker.touch();
return true;
}
});
controller.setLogPositionManager(new AbstractLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
Assert.assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
Assert.assertEquals(entryPosition, defaultPosition);
}
use of com.alibaba.otter.canal.parse.index.AbstractLogPositionManager in project canal by alibaba.
the class LocalBinlogEventParserTest method test_no_position.
@Test
public void test_no_position() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 1000);
final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", null, new Date().getTime() + 1000 * 1000L);
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
controller.setMasterPosition(defaultPosition);
controller.setMasterInfo(buildAuthentication());
controller.setDirectory(directory);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
for (Entry entry : entrys) {
entryCount.incrementAndGet();
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
controller.stop();
timeoutChecker.stop();
timeoutChecker.touch();
return true;
}
});
controller.setLogPositionManager(new AbstractLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
Assert.assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
// assertEquals(entryPosition.getJournalName(), "mysql-bin.000002");
Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}
Aggregations