Examples with OracleTriggerMonitoredSourceInfo com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo used on opensource projects

Example 6 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class BootstrapAvroFileSeederMain method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    File sourcesJson = new File(_sSourcesConfigFile);
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
    physicalSourceConfig.checkForNulls();
    Config config = new Config();
    ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
    _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    // Make sure the URI from the configuration file identifies an Oracle JDBC source.
    String uri = physicalSourceConfig.getUri();
    if (!uri.startsWith("jdbc:oracle")) {
        throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
    }
    OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
    // Parse each one of the logical sources
    _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
    FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
    for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
        OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
    }
    _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
    _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
    _sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
    _sReader = new BootstrapAvroFileEventReader(_sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sBootstrapBuffer);
}

Example 7 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class BootstrapSeederMain method init.

public static void init(String[] args) throws Exception {
    parseArgs(args);
    // Load the source configuration JSON file
    //File sourcesJson = new File("integration-test/config/sources-member2.json");
    File sourcesJson = new File(_sSourcesConfigFile);
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
    physicalSourceConfig.checkForNulls();
    Config config = new Config();
    ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
    _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
    // Make sure the URI from the configuration file identifies an Oracle JDBC source.
    String uri = physicalSourceConfig.getUri();
    if (!uri.startsWith("jdbc:oracle")) {
        throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
    }
    String sourceTypeStr = physicalSourceConfig.getReplBitSetter().getSourceType();
    if (SourceType.TOKEN.toString().equalsIgnoreCase(sourceTypeStr))
        throw new InvalidConfigException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
    // Create the OracleDataSource used to get DB connection(s)
    try {
        Class oracleDataSourceClass = OracleJarUtils.loadClass("oracle.jdbc.pool.OracleDataSource");
        Object ods = oracleDataSourceClass.newInstance();
        Method setURLMethod = oracleDataSourceClass.getMethod("setURL", String.class);
        setURLMethod.invoke(ods, uri);
        _sDataStore = (DataSource) ods;
    } catch (Exception e) {
        String errMsg = "Error creating a data source object ";
        LOG.error(errMsg, e);
        throw e;
    }
    //TODO: Need a better way than relaying on RelayFactory for generating MonitoredSourceInfo
    OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
    // Parse each one of the logical sources
    _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
    FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
    Set<String> seenUris = new HashSet<String>();
    for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
        String srcUri = sourceConfig.getUri();
        if (seenUris.contains(srcUri)) {
            String msg = "Uri (" + srcUri + ") is used for more than one sources. Currently Bootstrap Seeder cannot support seeding sources with the same URI together. Please have them run seperately !!";
            LOG.fatal(msg);
            throw new InvalidConfigException(msg);
        }
        seenUris.add(srcUri);
        OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
    }
    _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
    _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
    _sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
    _sReader = new BootstrapSrcDBEventReader(_sDataStore, _sBootstrapBuffer, _sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sSeeder.getLastKeys(), 0);
}

Example 8 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class BootstrapSrcDBEventReader method readEventsFromAllSources.

@Override
public ReadEventCycleSummary readEventsFromAllSources(long sinceSCN) throws DatabusException, EventCreationException, UnsupportedKeyException {
    List<EventReaderSummary> summaries = new ArrayList<EventReaderSummary>();
    long maxScn = EventReaderSummary.NO_EVENTS_SCN;
    long endScn = maxScn;
    boolean error = false;
    long startTS = System.currentTimeMillis();
    try {
        _rate.start();
        _rate.suspend();
        Connection conn = null;
        try {
            conn = _dataSource.getConnection();
            LOG.info("Oracle JDBC Version :" + conn.getMetaData().getDriverVersion());
        } finally {
            DBHelper.close(conn);
        }
        if (!_sources.isEmpty()) {
            // Script assumes seeding is done for one schema at a time
            // just use one source to get the schema name for sy$txlog
            maxScn = getMaxScn(_sources.get(0));
        }
        for (OracleTriggerMonitoredSourceInfo sourceInfo : _sources) {
            LOG.info("Bootstrapping " + sourceInfo.getEventView());
            _bootstrapSeedWriter.start(maxScn);
            EventReaderSummary summary = readEventsForSource(sourceInfo, maxScn);
            // Script assumes seeding is done for one schema at a time
            // just use one source to get the schema name for sy$txlog
            endScn = getMaxScn(_sources.get(0));
            _bootstrapSeedWriter.endEvents(BootstrapEventBuffer.END_OF_SOURCE, endScn, null);
            summaries.add(summary);
        }
    } catch (Exception ex) {
        error = true;
        throw new DatabusException(ex);
    } finally {
        // Notify writer that I am done
        if (error) {
            _bootstrapSeedWriter.endEvents(BootstrapEventBuffer.ERROR_CODE, endScn, null);
            LOG.error("Seeder stopping unexpectedly !!");
        } else {
            _bootstrapSeedWriter.endEvents(BootstrapEventBuffer.END_OF_FILE, endScn, null);
            LOG.info("Completed Seeding !!");
        }
        LOG.info("Start SCN :" + maxScn);
        LOG.info("End SCN :" + endScn);
    }
    long endTS = System.currentTimeMillis();
    ReadEventCycleSummary cycleSummary = new ReadEventCycleSummary("seeder", summaries, maxScn, (endTS - startTS));
    return cycleSummary;
}

Example 9 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class BootstrapDBSeeder method initSources.

public void initSources() throws SQLException {
    LOG.info("MySQL JDBC Version :" + getConnection().getMetaData().getDriverVersion());
    _lastRows = new HashMap<String, Long>();
    _lastKeys = new HashMap<String, String>();
    for (OracleTriggerMonitoredSourceInfo sourceInfo : _sources) {
        createBootStrapSourceDB(sourceInfo);
        _lastRows.put(sourceInfo.getEventView(), getRowIdFromCheckpoint(sourceInfo));
        String k = geSrcKeyFromCheckpoint(sourceInfo);
        _lastKeys.put(sourceInfo.getEventView(), k);
        if (-1 == _currSrcId)
            _currSrcId = sourceInfo.getSourceId();
        if (null == _lastSeenKey)
            _lastSeenKey = k;
    }
}

Example 10 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class BootstrapDBSeeder method createLogTableRows.

private void createLogTableRows() throws SQLException {
    String sql = "insert into bootstrap_loginfo (srcid, logid, minwindowscn, maxwindowscn, maxrid) values ( ?, 0, -1, -1, 0)";
    Connection conn = null;
    PreparedStatement stmt = null;
    try {
        conn = getConnection();
        stmt = conn.prepareStatement(sql);
        for (OracleTriggerMonitoredSourceInfo srcInfo : _sources) {
            try {
                stmt.setInt(1, srcInfo.getSourceId());
                stmt.executeUpdate();
                _bootstrapDao.createNewLogTable(srcInfo.getSourceId());
            } catch (SQLException sqlEx) {
                LOG.error("Got Error inserting entry into bootstrap_loginfo but proceeding !!", sqlEx);
            }
        }
    } catch (SQLException sqlEx) {
        LOG.error("Got Error inserting entry into bootstrap_loginfo !!", sqlEx);
        throw sqlEx;
    } finally {
        DBHelper.close(stmt);
    }
}