Examples with PGReplicationStream org.postgresql.replication.PGReplicationStream used on opensource projects

Example 1 with PGReplicationStream

use of org.postgresql.replication.PGReplicationStream in project debezium by debezium.

the class PostgresReplicationConnection method createReplicationStream.

private ReplicationStream createReplicationStream(final LogSequenceNumber lsn) throws SQLException {
    PGReplicationStream s;
    try {
        s = startPgReplicationStream(lsn, plugin.forceRds() ? messageDecoder::optionsWithoutMetadata : messageDecoder::optionsWithMetadata);
        messageDecoder.setContainsMetadata(plugin.forceRds() ? false : true);
    } catch (PSQLException e) {
        if (e.getMessage().matches("(?s)ERROR: option .* is unknown.*")) {
            // It is possible we are connecting to an old wal2json plug-in
            LOGGER.warn("Could not register for streaming with metadata in messages, falling back to messages without metadata");
            s = startPgReplicationStream(lsn, messageDecoder::optionsWithoutMetadata);
            messageDecoder.setContainsMetadata(false);
        } else if (e.getMessage().matches("(?s)ERROR: requested WAL segment .* has already been removed.*")) {
            LOGGER.error("Cannot rewind to last processed WAL position", e);
            throw new ConnectException("The offset to start reading from has been removed from the database write-ahead log. Create a new snapshot and consider setting of PostgreSQL parameter wal_keep_segments = 0.");
        } else {
            throw e;
        }
    }
    final PGReplicationStream stream = s;
    final long lsnLong = lsn.asLong();
    return new ReplicationStream() {

        private static final int CHECK_WARNINGS_AFTER_COUNT = 100;

        private int warningCheckCounter = CHECK_WARNINGS_AFTER_COUNT;

        // make sure this is volatile since multiple threads may be interested in this value
        private volatile LogSequenceNumber lastReceivedLSN;

        @Override
        public void read(ReplicationMessageProcessor processor) throws SQLException, InterruptedException {
            ByteBuffer read = stream.read();
            // the lsn we started from is inclusive, so we need to avoid sending back the same message twice
            if (lsnLong >= stream.getLastReceiveLSN().asLong()) {
                return;
            }
            deserializeMessages(read, processor);
        }

        @Override
        public void readPending(ReplicationMessageProcessor processor) throws SQLException, InterruptedException {
            ByteBuffer read = stream.readPending();
            // the lsn we started from is inclusive, so we need to avoid sending back the same message twice
            if (read == null || lsnLong >= stream.getLastReceiveLSN().asLong()) {
                return;
            }
            deserializeMessages(read, processor);
        }

        private void deserializeMessages(ByteBuffer buffer, ReplicationMessageProcessor processor) throws SQLException, InterruptedException {
            lastReceivedLSN = stream.getLastReceiveLSN();
            messageDecoder.processMessage(buffer, processor, typeRegistry);
        }

        @Override
        public void close() throws SQLException {
            processWarnings(true);
            stream.close();
        }

        @Override
        public void flushLastReceivedLsn() throws SQLException {
            if (lastReceivedLSN == null) {
                // nothing to flush yet, since we haven't read anything...
                return;
            }
            doFlushLsn(lastReceivedLSN);
        }

        @Override
        public void flushLsn(long lsn) throws SQLException {
            doFlushLsn(LogSequenceNumber.valueOf(lsn));
        }

        private void doFlushLsn(LogSequenceNumber lsn) throws SQLException {
            stream.setFlushedLSN(lsn);
            stream.setAppliedLSN(lsn);
            stream.forceUpdateStatus();
        }

        @Override
        public Long lastReceivedLsn() {
            return lastReceivedLSN != null ? lastReceivedLSN.asLong() : null;
        }

        private void processWarnings(final boolean forced) throws SQLException {
            if (--warningCheckCounter == 0 || forced) {
                warningCheckCounter = CHECK_WARNINGS_AFTER_COUNT;
                for (SQLWarning w = connection().getWarnings(); w != null; w = w.getNextWarning()) {
                    LOGGER.debug("Server-side message: '{}', state = {}, code = {}", w.getMessage(), w.getSQLState(), w.getErrorCode());
                }
            }
        }
    };
}

Example 2 with PGReplicationStream

use of org.postgresql.replication.PGReplicationStream in project debezium by debezium.

the class PostgresReplicationConnection method startPgReplicationStream.

private PGReplicationStream startPgReplicationStream(final LogSequenceNumber lsn, Function<ChainedLogicalStreamBuilder, ChainedLogicalStreamBuilder> configurator) throws SQLException {
    assert lsn != null;
    ChainedLogicalStreamBuilder streamBuilder = pgConnection().getReplicationAPI().replicationStream().logical().withSlotName(slotName).withStartPosition(lsn);
    streamBuilder = configurator.apply(streamBuilder);
    if (statusUpdateIntervalMillis != null && statusUpdateIntervalMillis > 0) {
        streamBuilder.withStatusInterval(statusUpdateIntervalMillis, TimeUnit.MILLISECONDS);
    }
    PGReplicationStream stream = streamBuilder.start();
    // Needed by tests when connections are opened and closed in a fast sequence
    try {
        Thread.sleep(10);
    } catch (Exception e) {
    }
    stream.forceUpdateStatus();
    return stream;
}