Examples with LogSequenceNumber org.postgresql.replication.LogSequenceNumber used on opensource projects

Example 1 with LogSequenceNumber

use of org.postgresql.replication.LogSequenceNumber in project debezium by debezium.

the class PostgresReplicationConnection method startStreaming.

@Override
public ReplicationStream startStreaming(Long offset) throws SQLException {
    connect();
    if (offset == null || offset <= 0) {
        offset = defaultStartingPos;
    }
    LogSequenceNumber lsn = LogSequenceNumber.valueOf(offset);
    LOGGER.debug("starting streaming from LSN '{}'", lsn.asString());
    return createReplicationStream(lsn);
}

Example 2 with LogSequenceNumber

use of org.postgresql.replication.LogSequenceNumber in project debezium by debezium.

the class PostgresReplicationConnection method createReplicationStream.

private ReplicationStream createReplicationStream(final LogSequenceNumber lsn) throws SQLException {
    PGReplicationStream s;
    try {
        s = startPgReplicationStream(lsn, plugin.forceRds() ? messageDecoder::optionsWithoutMetadata : messageDecoder::optionsWithMetadata);
        messageDecoder.setContainsMetadata(plugin.forceRds() ? false : true);
    } catch (PSQLException e) {
        if (e.getMessage().matches("(?s)ERROR: option .* is unknown.*")) {
            // It is possible we are connecting to an old wal2json plug-in
            LOGGER.warn("Could not register for streaming with metadata in messages, falling back to messages without metadata");
            s = startPgReplicationStream(lsn, messageDecoder::optionsWithoutMetadata);
            messageDecoder.setContainsMetadata(false);
        } else if (e.getMessage().matches("(?s)ERROR: requested WAL segment .* has already been removed.*")) {
            LOGGER.error("Cannot rewind to last processed WAL position", e);
            throw new ConnectException("The offset to start reading from has been removed from the database write-ahead log. Create a new snapshot and consider setting of PostgreSQL parameter wal_keep_segments = 0.");
        } else {
            throw e;
        }
    }
    final PGReplicationStream stream = s;
    final long lsnLong = lsn.asLong();
    return new ReplicationStream() {

        private static final int CHECK_WARNINGS_AFTER_COUNT = 100;

        private int warningCheckCounter = CHECK_WARNINGS_AFTER_COUNT;

        // make sure this is volatile since multiple threads may be interested in this value
        private volatile LogSequenceNumber lastReceivedLSN;

        @Override
        public void read(ReplicationMessageProcessor processor) throws SQLException, InterruptedException {
            ByteBuffer read = stream.read();
            // the lsn we started from is inclusive, so we need to avoid sending back the same message twice
            if (lsnLong >= stream.getLastReceiveLSN().asLong()) {
                return;
            }
            deserializeMessages(read, processor);
        }

        @Override
        public void readPending(ReplicationMessageProcessor processor) throws SQLException, InterruptedException {
            ByteBuffer read = stream.readPending();
            // the lsn we started from is inclusive, so we need to avoid sending back the same message twice
            if (read == null || lsnLong >= stream.getLastReceiveLSN().asLong()) {
                return;
            }
            deserializeMessages(read, processor);
        }

        private void deserializeMessages(ByteBuffer buffer, ReplicationMessageProcessor processor) throws SQLException, InterruptedException {
            lastReceivedLSN = stream.getLastReceiveLSN();
            messageDecoder.processMessage(buffer, processor, typeRegistry);
        }

        @Override
        public void close() throws SQLException {
            processWarnings(true);
            stream.close();
        }

        @Override
        public void flushLastReceivedLsn() throws SQLException {
            if (lastReceivedLSN == null) {
                // nothing to flush yet, since we haven't read anything...
                return;
            }
            doFlushLsn(lastReceivedLSN);
        }

        @Override
        public void flushLsn(long lsn) throws SQLException {
            doFlushLsn(LogSequenceNumber.valueOf(lsn));
        }

        private void doFlushLsn(LogSequenceNumber lsn) throws SQLException {
            stream.setFlushedLSN(lsn);
            stream.setAppliedLSN(lsn);
            stream.forceUpdateStatus();
        }

        @Override
        public Long lastReceivedLsn() {
            return lastReceivedLSN != null ? lastReceivedLSN.asLong() : null;
        }

        private void processWarnings(final boolean forced) throws SQLException {
            if (--warningCheckCounter == 0 || forced) {
                warningCheckCounter = CHECK_WARNINGS_AFTER_COUNT;
                for (SQLWarning w = connection().getWarnings(); w != null; w = w.getNextWarning()) {
                    LOGGER.debug("Server-side message: '{}', state = {}, code = {}", w.getMessage(), w.getSQLState(), w.getErrorCode());
                }
            }
        }
    };
}

Example 3 with LogSequenceNumber

use of org.postgresql.replication.LogSequenceNumber in project eventuate-local by eventuate-local.

the class PostgresWalClient method connectAndRun.

private void connectAndRun(Optional<BinlogFileOffset> binlogFileOffset, Consumer<EVENT> eventConsumer) throws SQLException, InterruptedException, IOException {
    countDownLatchForStop = new CountDownLatch(1);
    Properties props = new Properties();
    PGProperty.USER.set(props, user);
    PGProperty.PASSWORD.set(props, password);
    PGProperty.ASSUME_MIN_SERVER_VERSION.set(props, "9.4");
    PGProperty.REPLICATION.set(props, "database");
    PGProperty.PREFER_QUERY_MODE.set(props, "simple");
    connection = DriverManager.getConnection(url, props);
    PGConnection replConnection = connection.unwrap(PGConnection.class);
    LogSequenceNumber lsn = binlogFileOffset.flatMap(offset -> Optional.ofNullable(offset.getOffset()).map(LogSequenceNumber::valueOf)).orElse(LogSequenceNumber.valueOf("0/0"));
    stream = replConnection.getReplicationAPI().replicationStream().logical().withSlotName(replicationSlotName).withSlotOption("include-xids", false).withStatusInterval(replicationStatusIntervalInMilliseconds, TimeUnit.MILLISECONDS).withStartPosition(lsn).start();
    logger.info("connection to postgres wal succeed");
    while (running) {
        ByteBuffer messageBuffer = stream.readPending();
        if (messageBuffer == null) {
            logger.info("Got empty message, sleeping");
            TimeUnit.MILLISECONDS.sleep(walIntervalInMilliseconds);
            continue;
        }
        String messageString = extractStringFromBuffer(messageBuffer);
        logger.info("Got message: " + messageString);
        postgresWalMessageParser.parse(JSonMapper.fromJson(messageString, PostgresWalMessage.class), stream.getLastReceiveLSN().asLong(), replicationSlotName).forEach(eventConsumer);
        stream.setAppliedLSN(stream.getLastReceiveLSN());
        stream.setFlushedLSN(stream.getLastReceiveLSN());
    }
    try {
        stream.close();
        connection.close();
    } catch (SQLException e) {
        logger.error(e.getMessage(), e);
        throw new RuntimeException(e);
    }
    countDownLatchForStop.countDown();
}