Examples with EventHeaderV4 com.github.shyiko.mysql.binlog.event.EventHeaderV4 used on opensource projects

Example 1 with EventHeaderV4

use of com.github.shyiko.mysql.binlog.event.EventHeaderV4 in project debezium by debezium.

the class BinlogReader method handleServerIncident.

/**
 * Handle the supplied event that signals that an out of the ordinary event that occurred on the master. It notifies the slave
 * that something happened on the master that might cause data to be in an inconsistent state.
 *
 * @param event the server stopped event to be processed; may not be null
 */
protected void handleServerIncident(Event event) {
    if (event.getData() instanceof EventDataDeserializationExceptionData) {
        EventDataDeserializationExceptionData data = event.getData();
        // safe cast, instantiated that ourselves
        EventHeaderV4 eventHeader = (EventHeaderV4) data.getCause().getEventHeader();
        // logging some additional context but not the exception itself, this will happen in handleEvent()
        if (eventDeserializationFailureHandlingMode == EventProcessingFailureHandlingMode.FAIL) {
            logger.error("Error while deserializing binlog event at offset {}.{}" + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", source.offset(), System.lineSeparator(), eventHeader.getPosition(), eventHeader.getNextPosition(), source.binlogFilename());
            throw new RuntimeException(data.getCause());
        } else if (eventDeserializationFailureHandlingMode == EventProcessingFailureHandlingMode.WARN) {
            logger.warn("Error while deserializing binlog event at offset {}.{}" + "This exception will be ignored and the event be skipped.{}" + "Use the mysqlbinlog tool to view the problematic event: mysqlbinlog --start-position={} --stop-position={} --verbose {}", source.offset(), System.lineSeparator(), System.lineSeparator(), eventHeader.getPosition(), eventHeader.getNextPosition(), source.binlogFilename(), data.getCause());
        }
    } else {
        logger.error("Server incident: {}", event);
    }
}

Example 2 with EventHeaderV4

use of com.github.shyiko.mysql.binlog.event.EventHeaderV4 in project SpinalTap by airbnb.

the class MysqlSource method toBinlogEvent.

public static BinlogEvent toBinlogEvent(Event event, BinlogFilePos filePos) {
    EventHeaderV4 header = event.getHeader();
    EventType eventType = header.getEventType();
    long serverId = header.getServerId();
    long timestamp = header.getTimestamp();
    if (EventType.isWrite(eventType)) {
        WriteRowsEventData data = event.getData();
        return new WriteEvent(data.getTableId(), serverId, timestamp, filePos, data.getRows());
    } else if (EventType.isUpdate(eventType)) {
        UpdateRowsEventData data = event.getData();
        return new UpdateEvent(data.getTableId(), serverId, timestamp, filePos, data.getRows());
    } else if (EventType.isDelete(eventType)) {
        DeleteRowsEventData data = event.getData();
        return new DeleteEvent(data.getTableId(), serverId, timestamp, filePos, data.getRows());
    } else {
        switch(eventType) {
            case TABLE_MAP:
                TableMapEventData tableMapData = event.getData();
                return new TableMapEvent(tableMapData.getTableId(), serverId, timestamp, filePos, tableMapData.getDatabase(), tableMapData.getTable(), tableMapData.getColumnTypes());
            case XID:
                XidEventData xidData = event.getData();
                return new XidEvent(serverId, timestamp, filePos, xidData.getXid());
            case QUERY:
                QueryEventData queryData = event.getData();
                return new QueryEvent(serverId, timestamp, filePos, queryData.getDatabase(), queryData.getSql());
            case FORMAT_DESCRIPTION:
                return new StartEvent(serverId, timestamp, filePos);
            default:
                return null;
        }
    }
}

Example 3 with EventHeaderV4

use of com.github.shyiko.mysql.binlog.event.EventHeaderV4 in project nifi by apache.

the class CaptureChangeMySQL method outputEvents.

public void outputEvents(ProcessSession session, StateManager stateManager, ComponentLog log) throws IOException {
    RawBinlogEvent rawBinlogEvent;
    // Drain the queue
    while ((rawBinlogEvent = queue.poll()) != null && !doStop.get()) {
        Event event = rawBinlogEvent.getEvent();
        EventHeaderV4 header = event.getHeader();
        long timestamp = header.getTimestamp();
        EventType eventType = header.getEventType();
        // is filled in during the ROTATE case
        if (eventType != ROTATE && eventType != FORMAT_DESCRIPTION) {
            currentBinlogPosition = header.getPosition();
        }
        log.debug("Got message event type: {} ", new Object[] { header.getEventType().toString() });
        switch(eventType) {
            case TABLE_MAP:
                // This is sent to inform which table is about to be changed by subsequent events
                TableMapEventData data = event.getData();
                // Should we skip this table? Yes if we've specified a DB or table name pattern and they don't match
                skipTable = (databaseNamePattern != null && !databaseNamePattern.matcher(data.getDatabase()).matches()) || (tableNamePattern != null && !tableNamePattern.matcher(data.getTable()).matches());
                if (!skipTable) {
                    TableInfoCacheKey key = new TableInfoCacheKey(this.getIdentifier(), data.getDatabase(), data.getTable(), data.getTableId());
                    if (cacheClient != null) {
                        try {
                            currentTable = cacheClient.get(key, cacheKeySerializer, cacheValueDeserializer);
                        } catch (ConnectException ce) {
                            throw new IOException("Could not connect to Distributed Map Cache server to get table information", ce);
                        }
                        if (currentTable == null) {
                            // We don't have an entry for this table yet, so fetch the info from the database and populate the cache
                            try {
                                currentTable = loadTableInfo(key);
                                try {
                                    cacheClient.put(key, currentTable, cacheKeySerializer, cacheValueSerializer);
                                } catch (ConnectException ce) {
                                    throw new IOException("Could not connect to Distributed Map Cache server to put table information", ce);
                                }
                            } catch (SQLException se) {
                                // Propagate the error up, so things like rollback and logging/bulletins can be handled
                                throw new IOException(se.getMessage(), se);
                            }
                        }
                    }
                } else {
                    // Clear the current table, to force a reload next time we get a TABLE_MAP event we care about
                    currentTable = null;
                }
                break;
            case QUERY:
                QueryEventData queryEventData = event.getData();
                currentDatabase = queryEventData.getDatabase();
                String sql = queryEventData.getSql();
                // Is this the start of a transaction?
                if ("BEGIN".equals(sql)) {
                    // If we're already in a transaction, something bad happened, alert the user
                    if (inTransaction) {
                        throw new IOException("BEGIN event received while already processing a transaction. This could indicate that your binlog position is invalid.");
                    }
                    // Mark the current binlog position in case we have to rollback the transaction (if the processor is stopped, e.g.)
                    xactBinlogFile = currentBinlogFile;
                    xactBinlogPosition = currentBinlogPosition;
                    xactSequenceId = currentSequenceId.get();
                    if (includeBeginCommit && (databaseNamePattern == null || databaseNamePattern.matcher(currentDatabase).matches())) {
                        BeginTransactionEventInfo beginEvent = new BeginTransactionEventInfo(currentDatabase, timestamp, currentBinlogFile, currentBinlogPosition);
                        currentSequenceId.set(beginEventWriter.writeEvent(currentSession, transitUri, beginEvent, currentSequenceId.get(), REL_SUCCESS));
                    }
                    inTransaction = true;
                } else if ("COMMIT".equals(sql)) {
                    if (!inTransaction) {
                        throw new IOException("COMMIT event received while not processing a transaction (i.e. no corresponding BEGIN event). " + "This could indicate that your binlog position is invalid.");
                    }
                    // InnoDB generates XID events for "commit", but MyISAM generates Query events with "COMMIT", so handle that here
                    if (includeBeginCommit && (databaseNamePattern == null || databaseNamePattern.matcher(currentDatabase).matches())) {
                        CommitTransactionEventInfo commitTransactionEvent = new CommitTransactionEventInfo(currentDatabase, timestamp, currentBinlogFile, currentBinlogPosition);
                        currentSequenceId.set(commitEventWriter.writeEvent(currentSession, transitUri, commitTransactionEvent, currentSequenceId.get(), REL_SUCCESS));
                    }
                    // Commit the NiFi session
                    session.commit();
                    inTransaction = false;
                    currentTable = null;
                } else {
                    // Check for DDL events (alter table, e.g.). Normalize the query to do string matching on the type of change
                    String normalizedQuery = sql.toLowerCase().trim().replaceAll(" {2,}", " ");
                    if (normalizedQuery.startsWith("alter table") || normalizedQuery.startsWith("alter ignore table") || normalizedQuery.startsWith("create table") || normalizedQuery.startsWith("truncate table") || normalizedQuery.startsWith("rename table") || normalizedQuery.startsWith("drop table") || normalizedQuery.startsWith("drop database")) {
                        if (includeDDLEvents && (databaseNamePattern == null || databaseNamePattern.matcher(currentDatabase).matches())) {
                            // If we don't have table information, we can still use the database name
                            TableInfo ddlTableInfo = (currentTable != null) ? currentTable : new TableInfo(currentDatabase, null, null, null);
                            DDLEventInfo ddlEvent = new DDLEventInfo(ddlTableInfo, timestamp, currentBinlogFile, currentBinlogPosition, sql);
                            currentSequenceId.set(ddlEventWriter.writeEvent(currentSession, transitUri, ddlEvent, currentSequenceId.get(), REL_SUCCESS));
                        }
                        // Remove all the keys from the cache that this processor added
                        if (cacheClient != null) {
                            cacheClient.removeByPattern(this.getIdentifier() + ".*");
                        }
                        // If not in a transaction, commit the session so the DDL event(s) will be transferred
                        if (includeDDLEvents && !inTransaction) {
                            session.commit();
                        }
                    }
                }
                break;
            case XID:
                if (!inTransaction) {
                    throw new IOException("COMMIT event received while not processing a transaction (i.e. no corresponding BEGIN event). " + "This could indicate that your binlog position is invalid.");
                }
                if (includeBeginCommit && (databaseNamePattern == null || databaseNamePattern.matcher(currentDatabase).matches())) {
                    CommitTransactionEventInfo commitTransactionEvent = new CommitTransactionEventInfo(currentDatabase, timestamp, currentBinlogFile, currentBinlogPosition);
                    currentSequenceId.set(commitEventWriter.writeEvent(currentSession, transitUri, commitTransactionEvent, currentSequenceId.get(), REL_SUCCESS));
                }
                // Commit the NiFi session
                session.commit();
                inTransaction = false;
                currentTable = null;
                currentDatabase = null;
                break;
            case WRITE_ROWS:
            case EXT_WRITE_ROWS:
            case PRE_GA_WRITE_ROWS:
            case UPDATE_ROWS:
            case EXT_UPDATE_ROWS:
            case PRE_GA_UPDATE_ROWS:
            case DELETE_ROWS:
            case EXT_DELETE_ROWS:
            case PRE_GA_DELETE_ROWS:
                // If we are skipping this table, then don't emit any events related to its modification
                if (skipTable) {
                    break;
                }
                if (!inTransaction) {
                    // These events should only happen inside a transaction, warn the user otherwise
                    log.warn("Table modification event occurred outside of a transaction.");
                    break;
                }
                if (currentTable == null && cacheClient != null) {
                    // No Table Map event was processed prior to this event, which should not happen, so throw an error
                    throw new RowEventException("No table information is available for this event, cannot process further.");
                }
                if (eventType == WRITE_ROWS || eventType == EXT_WRITE_ROWS || eventType == PRE_GA_WRITE_ROWS) {
                    InsertRowsEventInfo eventInfo = new InsertRowsEventInfo(currentTable, timestamp, currentBinlogFile, currentBinlogPosition, event.getData());
                    currentSequenceId.set(insertRowsWriter.writeEvent(currentSession, transitUri, eventInfo, currentSequenceId.get(), REL_SUCCESS));
                } else if (eventType == DELETE_ROWS || eventType == EXT_DELETE_ROWS || eventType == PRE_GA_DELETE_ROWS) {
                    DeleteRowsEventInfo eventInfo = new DeleteRowsEventInfo(currentTable, timestamp, currentBinlogFile, currentBinlogPosition, event.getData());
                    currentSequenceId.set(deleteRowsWriter.writeEvent(currentSession, transitUri, eventInfo, currentSequenceId.get(), REL_SUCCESS));
                } else {
                    // Update event
                    UpdateRowsEventInfo eventInfo = new UpdateRowsEventInfo(currentTable, timestamp, currentBinlogFile, currentBinlogPosition, event.getData());
                    currentSequenceId.set(updateRowsWriter.writeEvent(currentSession, transitUri, eventInfo, currentSequenceId.get(), REL_SUCCESS));
                }
                break;
            case ROTATE:
                // Update current binlog filename
                RotateEventData rotateEventData = event.getData();
                currentBinlogFile = rotateEventData.getBinlogFilename();
                currentBinlogPosition = rotateEventData.getBinlogPosition();
                break;
            default:
                break;
        }
        // advance the position if it is not that type of event.
        if (eventType != ROTATE && eventType != FORMAT_DESCRIPTION) {
            currentBinlogPosition = header.getNextPosition();
        }
    }
}

Example 4 with EventHeaderV4

use of com.github.shyiko.mysql.binlog.event.EventHeaderV4 in project rocketmq-externals by apache.

the class EventProcessor method processXidEvent.

private void processXidEvent(Event event) {
    EventHeaderV4 header = event.getHeader();
    XidEventData data = event.getData();
    String binlogFilename = binaryLogClient.getBinlogFilename();
    Long position = header.getNextPosition();
    Long xid = data.getXid();
    BinlogPosition binlogPosition = new BinlogPosition(binlogFilename, position);
    transaction.setNextBinlogPosition(binlogPosition);
    transaction.setXid(xid);
    replicator.commit(transaction, true);
    transaction = new Transaction(config);
}

Example 5 with EventHeaderV4

use of com.github.shyiko.mysql.binlog.event.EventHeaderV4 in project debezium by debezium.

the class BinlogReader method handleEvent.

protected void handleEvent(Event event) {
    if (event == null)
        return;
    // Update the source offset info. Note that the client returns the value in *milliseconds*, even though the binlog
    // contains only *seconds* precision ...
    EventHeader eventHeader = event.getHeader();
    // client returns milliseconds, but only second
    source.setBinlogTimestampSeconds(eventHeader.getTimestamp() / 1000L);
    // precision
    source.setBinlogServerId(eventHeader.getServerId());
    EventType eventType = eventHeader.getEventType();
    if (eventType == EventType.ROTATE) {
        EventData eventData = event.getData();
        RotateEventData rotateEventData;
        if (eventData instanceof EventDeserializer.EventDataWrapper) {
            rotateEventData = (RotateEventData) ((EventDeserializer.EventDataWrapper) eventData).getInternal();
        } else {
            rotateEventData = (RotateEventData) eventData;
        }
        source.setBinlogStartPoint(rotateEventData.getBinlogFilename(), rotateEventData.getBinlogPosition());
    } else if (eventHeader instanceof EventHeaderV4) {
        EventHeaderV4 trackableEventHeader = (EventHeaderV4) eventHeader;
        source.setEventPosition(trackableEventHeader.getPosition(), trackableEventHeader.getEventLength());
    }
    // If there is a handler for this event, forward the event to it ...
    try {
        // Forward the event to the handler ...
        eventHandlers.getOrDefault(eventType, this::ignoreEvent).accept(event);
        // Generate heartbeat message if the time is right
        heartbeat.heartbeat((BlockingConsumer<SourceRecord>) this::enqueueRecord);
        // Capture that we've completed another event ...
        source.completeEvent();
        if (skipEvent) {
            // We're in the mode of skipping events and we just skipped this one, so decrement our skip count ...
            --initialEventsToSkip;
            skipEvent = initialEventsToSkip > 0;
        }
    } catch (RuntimeException e) {
        // There was an error in the event handler, so propagate the failure to Kafka Connect ...
        logReaderState();
        failed(e, "Error processing binlog event");
        // Do not stop the client, since Kafka Connect should stop the connector on it's own
        // (and doing it here may cause problems the second time it is stopped).
        // We can clear the listeners though so that we ignore all future events ...
        eventHandlers.clear();
        logger.info("Error processing binlog event, and propagating to Kafka Connect so it stops this connector. Future binlog events read before connector is shutdown will be ignored.");
    } catch (InterruptedException e) {
        // Most likely because this reader was stopped and our thread was interrupted ...
        Thread.interrupted();
        eventHandlers.clear();
        logger.info("Stopped processing binlog events due to thread interruption");
    }
}