Examples with BinlogFile com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile used on opensource projects

Example 1 with BinlogFile

use of com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile in project canal by alibaba.

the class RdsLocalBinlogEventParser method start.

public void start() throws CanalParseException {
    try {
        Assert.notNull(accesskey);
        Assert.notNull(secretkey);
        Assert.notNull(instanceId);
        Assert.notNull(url);
        Assert.notNull(directory);
        if (endTime == null) {
            endTime = System.currentTimeMillis();
        }
        EntryPosition entryPosition = findStartPosition(null);
        if (entryPosition == null) {
            throw new PositionNotFoundException("position not found!");
        }
        Long startTimeInMill = entryPosition.getTimestamp();
        if (startTimeInMill == null || startTimeInMill <= 0) {
            throw new PositionNotFoundException("position timestamp is empty!");
        }
        startTime = startTimeInMill;
        List<BinlogFile> binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url, accesskey, secretkey, instanceId, new Date(startTime), new Date(endTime));
        if (binlogFiles.isEmpty()) {
            throw new CanalParseException("start timestamp : " + startTimeInMill + " binlog files is empty");
        }
        binlogDownloadQueue = new BinlogDownloadQueue(binlogFiles, batchFileSize, directory);
        binlogDownloadQueue.silenceDownload();
        needWait = true;
        // try to download one file,use to test server id
        binlogDownloadQueue.tryOne();
    } catch (Throwable e) {
        logger.error("download binlog failed", e);
        throw new CanalParseException(e);
    }
    setParserExceptionHandler(this::handleMysqlParserException);
    super.start();
}

Example 2 with BinlogFile

use of com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile in project canal by alibaba.

the class BinlogDownloadQueue method tryOne.

public BinlogFile tryOne() throws Throwable {
    BinlogFile binlogFile = binlogList.poll();
    if (binlogFile == null) {
        throw new CanalParseException("download binlog is null");
    }
    download(binlogFile);
    hostId = binlogFile.getHostInstanceID();
    this.currentSize++;
    return binlogFile;
}

Example 3 with BinlogFile

use of com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile in project canal by alibaba.

the class BinlogDownloadQueue method prepare.

public void prepare() throws InterruptedException {
    for (int i = this.currentSize; i < batchFileSize && !binlogList.isEmpty(); i++) {
        BinlogFile binlogFile = null;
        while (!binlogList.isEmpty()) {
            binlogFile = binlogList.poll();
            if (!binlogFile.getHostInstanceID().equalsIgnoreCase(hostId)) {
                continue;
            }
            break;
        }
        if (binlogFile == null) {
            break;
        }
        this.downloadQueue.put(binlogFile);
        this.lastDownload = "mysql-bin." + binlogFile.getFileName();
        this.currentSize++;
    }
}

Example 4 with BinlogFile

use of com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile in project canal by alibaba.

the class BinlogDownloadQueue method download.

private void download(BinlogFile binlogFile) throws Throwable {
    String downloadLink = binlogFile.getDownloadLink();
    String fileName = binlogFile.getFileName();
    downloadLink = downloadLink.trim();
    CloseableHttpClient httpClient = null;
    if (downloadLink.startsWith("https")) {
        HttpClientBuilder builder = HttpClientBuilder.create();
        builder.setMaxConnPerRoute(50);
        builder.setMaxConnTotal(100);
        // 创建支持忽略证书的https
        final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, (x509Certificates, s) -> true).build();
        httpClient = HttpClientBuilder.create().setSSLContext(sslContext).setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create().register("http", PlainConnectionSocketFactory.INSTANCE).register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE)).build())).build();
    } else {
        httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
    }
    HttpGet httpGet = new HttpGet(downloadLink);
    RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(TIMEOUT).setConnectionRequestTimeout(TIMEOUT).setSocketTimeout(TIMEOUT).build();
    httpGet.setConfig(requestConfig);
    HttpResponse response = httpClient.execute(httpGet);
    int statusCode = response.getStatusLine().getStatusCode();
    if (statusCode != HttpResponseStatus.OK.code()) {
        throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:" + statusCode);
    }
    saveFile(new File(destDir), "mysql-bin." + fileName, response);
}

Example 5 with BinlogFile

use of com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile in project canal by alibaba.

the class BinlogDownloadQueue method prepareBinlogList.

private void prepareBinlogList() {
    for (BinlogFile binlog : this.binlogList) {
        String fileName = StringUtils.substringBetween(binlog.getDownloadLink(), "mysql-bin.", "?");
        binlog.setFileName(fileName);
    }
    this.binlogList.sort(Comparator.comparing(BinlogFile::getFileName));
}