Examples with OracleTriggerMonitoredSourceInfo com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo used on opensource projects

Example 1 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class MonitoringEventProducer method run.

@Override
public void run() {
    //check state and behave accordingly
    if (createDataSource() && openDbConn()) {
        do {
            PreparedStatement pstmt = null;
            ResultSet rs = null;
            try {
                long maxDBScn = getMaxTxlogSCN(_con);
                _log.info("Max DB Scn =  " + maxDBScn);
                _dbStats.setMaxDBScn(maxDBScn);
                for (OracleTriggerMonitoredSourceInfo source : _sources) {
                    String eventQuery = _monitorQueriesBySource.get(source.getSourceId());
                    pstmt = _con.prepareStatement(eventQuery);
                    pstmt.setFetchSize(10);
                    //get max scn - exactly one row;
                    rs = pstmt.executeQuery();
                    if (rs.next()) {
                        long maxScn = rs.getLong(1);
                        _log.info("Source: " + source.getSourceId() + " Max Scn=" + maxScn);
                        _dbStats.setSrcMaxScn(source.getSourceName(), maxScn);
                    }
                    DBHelper.commit(_con);
                    DBHelper.close(rs, pstmt, null);
                    if (_state != MonitorState.SHUT) {
                        Thread.sleep(PER_SRC_MAX_SCN_POLL_TIME);
                    }
                }
                if (_state != MonitorState.SHUT) {
                    Thread.sleep(MAX_SCN_POLL_TIME);
                }
            } catch (InterruptedException e) {
                _log.error("Exception trace", e);
                shutDown();
            } catch (SQLException e) {
                try {
                    DBHelper.rollback(_con);
                } catch (SQLException s) {
                }
                _log.error("Exception trace", e);
                shutDown();
            } finally {
                DBHelper.close(rs, pstmt, null);
            }
        } while (_state != MonitorState.SHUT);
        _log.info("Shutting down dbMonitor thread");
        DBHelper.close(_con);
    }
}

Example 2 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class OracleEventProducerFactory method buildEventProducer.

public EventProducer buildEventProducer(PhysicalSourceStaticConfig physicalSourceConfig, SchemaRegistryService schemaRegistryService, DbusEventBufferAppendable dbusEventBuffer, MBeanServer mbeanServer, DbusEventsStatisticsCollector dbusEventsStatisticsCollector, MaxSCNReaderWriter _maxScnReaderWriter) throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException, InvalidConfigException {
    // Make sure the URI from the configuration file identifies an Oracle JDBC source.
    String uri = physicalSourceConfig.getUri();
    if (!uri.startsWith("jdbc:oracle")) {
        throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
    }
    // Parse each one of the logical sources
    List<OracleTriggerMonitoredSourceInfo> sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
    for (LogicalSourceStaticConfig sourceConfig : physicalSourceConfig.getSources()) {
        OracleTriggerMonitoredSourceInfo source = buildOracleMonitoredSourceInfo(sourceConfig, physicalSourceConfig, schemaRegistryService);
        sources.add(source);
    }
    DataSource ds = null;
    try {
        ds = OracleJarUtils.createOracleDataSource(uri);
    } catch (Exception e) {
        String errMsg = "Oracle URI likely not supported. Trouble creating OracleDataSource";
        _log.error(errMsg);
        throw new InvalidConfigException(errMsg + e.getMessage());
    }
    // Create the event producer
    EventProducer eventProducer = new OracleEventProducer(sources, ds, dbusEventBuffer, true, dbusEventsStatisticsCollector, _maxScnReaderWriter, physicalSourceConfig, ManagementFactory.getPlatformMBeanServer());
    _log.info("Created OracleEventProducer for config:  " + physicalSourceConfig + " with slowSourceQueryThreshold = " + physicalSourceConfig.getSlowSourceQueryThreshold());
    return eventProducer;
}

Example 3 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class OracleEventProducerFactory method buildOracleMonitoredSourceInfo.

public OracleTriggerMonitoredSourceInfo buildOracleMonitoredSourceInfo(LogicalSourceStaticConfig sourceConfig, PhysicalSourceStaticConfig pConfig, SchemaRegistryService schemaRegistryService) throws DatabusException, EventCreationException, UnsupportedKeyException, InvalidConfigException {
    String schema = null;
    try {
        schema = schemaRegistryService.fetchLatestSchemaBySourceName(sourceConfig.getName());
    } catch (NoSuchSchemaException e) {
        throw new InvalidConfigException("Unable to load the schema for source (" + sourceConfig.getName() + ").");
    }
    if (schema == null) {
        throw new InvalidConfigException("Unable to load the schema for source (" + sourceConfig.getName() + ").");
    }
    _log.info("Loading schema for source id " + sourceConfig.getId() + ": " + schema);
    String eventViewSchema;
    String eventView;
    if (sourceConfig.getUri().indexOf('.') != -1) {
        String[] parts = sourceConfig.getUri().split("\\.");
        eventViewSchema = parts[0];
        eventView = parts[1];
    } else {
        eventViewSchema = null;
        eventView = sourceConfig.getUri();
    }
    if (eventView.toLowerCase().startsWith("sy$")) {
        eventView = eventView.substring(3);
    }
    PartitionFunction partitionFunction = buildPartitionFunction(sourceConfig);
    EventFactory factory = createEventFactory(eventViewSchema, eventView, sourceConfig, pConfig, schema, partitionFunction);
    EventSourceStatistics statisticsBean = new EventSourceStatistics(sourceConfig.getName());
    OracleTriggerMonitoredSourceInfo sourceInfo = new OracleTriggerMonitoredSourceInfo(sourceConfig.getId(), sourceConfig.getName(), eventViewSchema, eventView, factory, statisticsBean, sourceConfig.getRegularQueryHints(), sourceConfig.getChunkedTxnQueryHints(), sourceConfig.getChunkedScnQueryHints(), sourceConfig.isSkipInfinityScn());
    return sourceInfo;
}

Example 4 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class BootstrapAvroFileEventReader method readEventsFromHadoopFiles.

private EventReaderSummary readEventsFromHadoopFiles(OracleTriggerMonitoredSourceInfo sourceInfo, File avroSeedDir, Long windowSCN) {
    DataFileReader<GenericRecord> reader = null;
    File[] files = avroSeedDir.listFiles();
    List<File> fileList = Arrays.asList(files);
    Collections.sort(fileList);
    long numRead = 0;
    long prevNumRead = 0;
    long numBytes = 0;
    long timestamp = System.currentTimeMillis();
    long timeStart = timestamp;
    long lastTime = timestamp;
    long commitInterval = _config.getCommitInterval();
    long totLatency = 0;
    GenericRecord record = null;
    RateMonitor seedingRate = new RateMonitor("Seeding Rate");
    seedingRate.start();
    seedingRate.suspend();
    long startRowId = _lastRows.get(sourceInfo.getEventView());
    LOG.info("Last Known Row Id is :" + startRowId);
    boolean resumeSeedingRate = true;
    for (File avroSeedFile : files) {
        if (!avroSeedFile.isFile())
            continue;
        LOG.info("Seeding from File : " + avroSeedFile);
        try {
            reader = new DataFileReader<GenericRecord>(avroSeedFile, new GenericDatumReader<GenericRecord>());
        } catch (IOException e) {
            LOG.fatal("Failed to bootstrap from file " + avroSeedFile.getAbsolutePath(), e);
            throw new RuntimeException("Failed to bootstrap from file " + avroSeedFile.getAbsolutePath(), e);
        }
        try {
            boolean committed = false;
            for (GenericRecord hdfsRecord : reader) {
                record = hdfsRecord;
                committed = false;
                numRead++;
                if (numRead < startRowId)
                    continue;
                if (resumeSeedingRate) {
                    seedingRate.resume();
                    resumeSeedingRate = false;
                }
                seedingRate.tick();
                //LOG.info("Read record :" + record);	    			
                long start = System.nanoTime();
                long eventSize = sourceInfo.getFactory().createAndAppendEvent(windowSCN, timestamp, hdfsRecord, _bootstrapEventBuffer, false, null);
                numBytes += eventSize;
                long latency = System.nanoTime() - start;
                totLatency += latency;
                if (numRead % commitInterval == 0) {
                    _bootstrapEventBuffer.endEvents(numRead, timestamp, null);
                    _bootstrapEventBuffer.startEvents();
                    long procTime = totLatency / 1000000000;
                    long currTime = System.currentTimeMillis();
                    long diff = (currTime - lastTime) / 1000;
                    long timeSinceStart = (currTime - timeStart) / 1000;
                    LOG.info("Processed " + commitInterval + " rows in " + diff + " seconds, Avro Processing Time (seconds) so far :" + (procTime) + ",Seconds elapsed since start :" + (timeSinceStart) + ",Overall Row Rate:" + seedingRate.getRate() + ", NumRows Fetched so far:" + numRead + ". TotalEventSize :" + numBytes);
                    lastTime = currTime;
                    seedingRate.resume();
                    committed = true;
                }
            }
            if (!committed) {
                _bootstrapEventBuffer.endEvents(numRead, timestamp, null);
                _bootstrapEventBuffer.startEvents();
                long procTime = totLatency / 1000000000;
                long currTime = System.currentTimeMillis();
                long diff = (currTime - lastTime) / 1000;
                long timeSinceStart = (currTime - timeStart) / 1000;
                LOG.info("Completed Seeding from : " + avroSeedFile + ", Processed " + commitInterval + " rows in " + diff + " seconds, Avro Processing Time (seconds) so far :" + (procTime) + ",Seconds elapsed since start :" + (timeSinceStart) + ",Overall Row Rate:" + seedingRate.getRate() + ", NumRows Fetched so far:" + numRead + ". TotalEventSize :" + numBytes);
                lastTime = currTime;
                seedingRate.resume();
            }
        } catch (Exception e) {
            LOG.fatal("NumRead :" + numRead + ", Got Exception while processing generic record :" + record, e);
            throw new RuntimeException(e);
        }
        LOG.info("Processed " + (numRead - prevNumRead) + " rows of Source: " + sourceInfo.getSourceName() + " from file " + avroSeedFile);
        prevNumRead = numRead;
    }
    long timeEnd = System.currentTimeMillis();
    long elapsedMin = (timeEnd - timeStart) / (MILLISEC_TO_MIN);
    LOG.info("Processed " + numRead + " rows of Source: " + sourceInfo.getSourceName() + " in " + elapsedMin + " minutes");
    return new EventReaderSummary(sourceInfo.getSourceId(), sourceInfo.getSourceName(), -1, (int) numRead, numBytes, (timeEnd - timeStart), (timeEnd - timeStart) / numRead, 0, 0, 0);
}

Example 5 with OracleTriggerMonitoredSourceInfo

use of com.linkedin.databus2.producers.db.OracleTriggerMonitoredSourceInfo in project databus by linkedin.

the class BootstrapAvroFileEventReader method readEventsFromAllSources.

@Override
public ReadEventCycleSummary readEventsFromAllSources(long sinceSCN) throws DatabusException, EventCreationException, UnsupportedKeyException {
    List<EventReaderSummary> summaries = new ArrayList<EventReaderSummary>();
    boolean error = false;
    long startTS = System.currentTimeMillis();
    long endScn = -1;
    long minScn = Long.MAX_VALUE;
    try {
        for (OracleTriggerMonitoredSourceInfo sourceInfo : _sources) {
            endScn = _config.getSeedWindowSCNMap().get(sourceInfo.getEventView());
            minScn = Math.min(endScn, minScn);
            LOG.info("Bootstrapping " + sourceInfo.getEventView());
            _bootstrapEventBuffer.start(endScn);
            String dir = _config.getAvroSeedInputDirMap().get(sourceInfo.getEventView());
            File d = new File(dir);
            EventReaderSummary summary = readEventsFromHadoopFiles(sourceInfo, d, endScn);
            // Script assumes seeding is done for one schema at a time
            _bootstrapEventBuffer.endEvents(BootstrapEventBuffer.END_OF_SOURCE, endScn, null);
            summaries.add(summary);
        }
    } catch (Exception ex) {
        error = true;
        throw new DatabusException(ex);
    } finally {
        // Notify writer that I am done
        if (error) {
            _bootstrapEventBuffer.endEvents(BootstrapEventBuffer.ERROR_CODE, endScn, null);
            LOG.error("Seeder stopping unexpectedly !!");
        } else {
            _bootstrapEventBuffer.endEvents(BootstrapEventBuffer.END_OF_FILE, endScn, null);
            LOG.info("Completed Seeding !!");
        }
    }
    LOG.info("Start SCN :" + minScn);
    long endTS = System.currentTimeMillis();
    ReadEventCycleSummary cycleSummary = new ReadEventCycleSummary("seeder", summaries, minScn, (endTS - startTS));
    return cycleSummary;
}