Examples with BootstrapProcessingException com.linkedin.databus.bootstrap.api.BootstrapProcessingException used on opensource projects

Example 6 with BootstrapProcessingException

use of com.linkedin.databus.bootstrap.api.BootstrapProcessingException in project databus by linkedin.

the class TestBootstrapSCNProcessor method testShouldBypassSnapshot.

@Test
public void testShouldBypassSnapshot() throws SQLException, BootstrapProcessingException, NoSuchFieldException, IllegalAccessException, NoSuchMethodException {
    BootstrapSCNProcessor bsp = new BootstrapSCNProcessor();
    Long defaultRowsThresholdForSnapshotBypass = Long.MAX_VALUE;
    Map<String, Long> rowsThresholdForSnapshotBypass = new HashMap<String, Long>();
    Map<String, Boolean> disableSnapshotBypass = new HashMap<String, Boolean>();
    boolean predicatePushDown = false;
    Map<String, Boolean> predicatePushDownBypass = new HashMap<String, Boolean>();
    int queryTimeoutInSec = 10;
    BootstrapReadOnlyConfig db = null;
    boolean enableMinScnCheck = false;
    final long longestDbTxnTimeMins = 240L;
    BootstrapServerStaticConfig bssc = new BootstrapServerStaticConfig(defaultRowsThresholdForSnapshotBypass, rowsThresholdForSnapshotBypass, disableSnapshotBypass, predicatePushDown, predicatePushDownBypass, queryTimeoutInSec, enableMinScnCheck, db, longestDbTxnTimeMins);
    Field field = bsp.getClass().getDeclaredField("_config");
    field.setAccessible(true);
    field.set(bsp, bssc);
    int srcId = 101;
    long sinceScn = 5;
    long startScn = 10;
    BootstrapDBMetaDataDAO bmdd = EasyMock.createMock(BootstrapDBMetaDataDAO.class);
    EasyMock.expect(bmdd.getLogIdToCatchup(srcId, startScn)).andReturn(0).anyTimes();
    EasyMock.expect(bmdd.getLogIdToCatchup(srcId, sinceScn)).andReturn(0).anyTimes();
    EasyMock.replay(bmdd);
    Field dbDaoField = bsp.getClass().getDeclaredField("_dbDao");
    dbDaoField.setAccessible(true);
    dbDaoField.set(bsp, bmdd);
    List<SourceStatusInfo> srcList = new ArrayList<SourceStatusInfo>();
    String name = "foo";
    SourceStatusInfo ssi = new SourceStatusInfo(name, srcId, 4);
    srcList.add(ssi);
    // case 1. Single source, defaultRowsThresholdForSnapshotBypass set to Long.MAX_VALUE,
    // individual overrides not set
    boolean sbs = bsp.shouldBypassSnapshot(sinceScn, startScn, srcList);
    Assert.assertEquals(true, sbs);
    // case 2. Single source, defaultRowsThresholdForSnapshotBypass set to finite value,
    // individual overrides set for the source
    rowsThresholdForSnapshotBypass.put(name, Long.MAX_VALUE);
    BootstrapServerStaticConfig bssc2 = new BootstrapServerStaticConfig(defaultRowsThresholdForSnapshotBypass, rowsThresholdForSnapshotBypass, disableSnapshotBypass, predicatePushDown, predicatePushDownBypass, queryTimeoutInSec, enableMinScnCheck, db, longestDbTxnTimeMins);
    field.set(bsp, bssc2);
    sbs = bsp.shouldBypassSnapshot(sinceScn, startScn, srcList);
    Assert.assertEquals(true, sbs);
    // Case 3:  Detect case when the log is not available on log tables ( so it should NOT bypass snapshot )
    BootstrapDBMetaDataDAO bmdd2 = EasyMock.createMock(BootstrapDBMetaDataDAO.class);
    EasyMock.expect(bmdd2.getLogIdToCatchup(srcId, startScn)).andReturn(2).anyTimes();
    EasyMock.expect(bmdd2.getLogIdToCatchup(srcId, sinceScn)).andThrow(new BootstrapProcessingException(""));
    EasyMock.replay(bmdd2);
    Field dbDaoField2 = bsp.getClass().getDeclaredField("_dbDao");
    dbDaoField2.setAccessible(true);
    dbDaoField2.set(bsp, bmdd2);
    sbs = bsp.shouldBypassSnapshot(sinceScn, startScn, srcList);
    Assert.assertEquals(false, sbs);
}

Example 7 with BootstrapProcessingException

use of com.linkedin.databus.bootstrap.api.BootstrapProcessingException in project databus by linkedin.

the class EventProcessor method onEvent.

public BootstrapEventProcessResult onEvent(ResultSet rs, DbusEventsStatisticsCollector statsCollector) throws BootstrapProcessingException {
    try {
        rIds.add(rs.getLong(1));
        sequences.add(rs.getLong(2));
        srcKeys.add(rs.getString(3));
        values.add(rs.getBytes(4));
    } catch (Exception ex) {
        throw new BootstrapProcessingException(ex);
    }
    return new BootstrapEventProcessResult(1, false, false);
}

Example 8 with BootstrapProcessingException

use of com.linkedin.databus.bootstrap.api.BootstrapProcessingException in project databus by linkedin.

the class BootstrapEventWriter method onEvent.

@Override
public BootstrapEventProcessResult onEvent(ResultSet rs, DbusEventsStatisticsCollector statsCollector) throws BootstrapProcessingException {
    long rid = -1;
    boolean exceededBufferLimit = false;
    boolean dropped = true;
    try {
        if (null == _event) {
            ByteBuffer tmpBuffer = ByteBuffer.wrap(rs.getBytes(4));
            if (_debug) {
                LOG.debug("BUFFER SIZE:" + tmpBuffer.limit());
            }
            _event = _eventFactory.createReadOnlyDbusEventFromBuffer(tmpBuffer, tmpBuffer.position());
        } else {
            ByteBuffer tmpBuffer = ByteBuffer.wrap(rs.getBytes(4));
            if (_debug) {
                LOG.debug("Resized BUFFER SIZE:" + tmpBuffer.limit());
            }
            _event = _event.reset(tmpBuffer, 0);
        }
        if (_debug) {
            LOG.debug("Event fetched: " + _event.size() + " for source:" + _event.srcId());
        }
        if (!_event.isValid()) {
            LOG.error("got an error event :" + _event.toString());
            return BootstrapEventProcessResult.getFailedEventProcessingResult(_numRowsWritten);
        }
        rid = rs.getLong(1);
        if (_debug) {
            LOG.debug("sending: " + _event.getDbusEventKey() + " " + _event.sequence());
            LOG.debug("event size:" + _event.size());
        }
        if ((null == _filter) || (_filter.allow(_event))) {
            if (_debug) {
                if (null != _filter) {
                    LOG.debug("Event :" + _event.getDbusEventKey() + " passed filter check !!");
                }
            }
            // client has enough space for this event
            if (_bytesSent + _event.size() < _clientFreeBufferSize) {
                int sentBytes = _event.writeTo(_writeChannel, _encoding);
                // On exception, sentBytes are set to 0
                if (0 >= sentBytes) {
                    // to avoid successive write failures !!
                    return BootstrapEventProcessResult.getFailedEventProcessingResult(_numRowsWritten);
                }
                _bytesSent += sentBytes;
                // tracks processed Rows only
                _numRowsWritten++;
                dropped = false;
                if (_debug) {
                    LOG.debug("SENT " + _bytesSent);
                }
                if (null != statsCollector) {
                    statsCollector.registerDataEvent(_event);
                    if (_debug) {
                        LOG.debug("Stats NumEvents :" + statsCollector.getTotalStats().getNumDataEvents());
                    }
                }
            } else {
                exceededBufferLimit = true;
                _sizeOfPendingEvent = _event.size();
                LOG.info("Terminating batch with max. size of " + _clientFreeBufferSize + "; Bytes sent in the current batch is " + _bytesSent + "; Rows processed in the batch is " + _numRowsWritten + ((_numRowsWritten <= 0) ? ", Pending Event Size is : " + _sizeOfPendingEvent : ""));
            }
        } else {
            if (null != statsCollector) {
                statsCollector.registerDataEventFiltered(_event);
                if (_debug) {
                    LOG.debug("Stats NumFilteredEvents :" + statsCollector.getTotalStats().getNumDataEventsFiltered());
                }
                if (_debug) {
                    LOG.debug("Event :" + _event.getDbusEventKey() + " failed filter check !!");
                }
            }
        }
    } catch (SQLException e) {
        LOG.error("SQLException encountered while sending to client row " + rid);
        throw new BootstrapProcessingException(e);
    }
    return new BootstrapEventProcessResult(_numRowsWritten, exceededBufferLimit, dropped);
}

Example 9 with BootstrapProcessingException

use of com.linkedin.databus.bootstrap.api.BootstrapProcessingException in project databus by linkedin.

the class BootstrapRequestProcessor method doProcess.

/*
   * (non-Javadoc)
   *
   * @see
   * com.linkedin.databus.container.request.RequestProcessor#process(com.linkedin.databus
   * .container.request.DatabusRequest)
   */
@Override
protected DatabusRequest doProcess(DatabusRequest request) throws IOException, RequestProcessingException {
    BootstrapProcessor processor = null;
    BootstrapHttpStatsCollector bootstrapStatsCollector = _bootstrapServer.getBootstrapStatsCollector();
    long startTime = System.currentTimeMillis();
    boolean isDebug = LOG.isDebugEnabled();
    try {
        try {
            String threadName = Thread.currentThread().getName();
            DbusEventsStatisticsCollector threadCollector = _bootstrapServer.getOutBoundStatsCollectors().getStatsCollector(threadName);
            if (null == threadCollector) {
                threadCollector = new DbusEventsStatisticsCollector(_bootstrapServer.getContainerStaticConfig().getId(), threadName, true, false, _bootstrapServer.getMbeanServer());
                StatsCollectors<DbusEventsStatisticsCollector> ds = _bootstrapServer.getOutBoundStatsCollectors();
                ds.addStatsCollector(threadName, threadCollector);
            }
            processor = new BootstrapProcessor(_config, threadCollector);
        } catch (Exception e) {
            if (null != bootstrapStatsCollector) {
                bootstrapStatsCollector.registerErrBootstrap();
            }
            throw new RequestProcessingException(e);
        }
        DatabusComponentStatus componentStatus = _componentStatus.getStatusSnapshot();
        if (!componentStatus.isRunningStatus()) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(componentStatus.getMessage());
        }
        String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);
        DbusKeyFilter keyFilter = null;
        if ((null != partitionInfoString) && (!partitionInfoString.isEmpty())) {
            try {
                keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
                if (isDebug)
                    LOG.debug("ServerSideFilter is :" + keyFilter);
            } catch (Exception ex) {
                String msg = "Unable to parse partitionInfo from request. PartitionInfo was :" + partitionInfoString;
                LOG.error(msg, ex);
                throw new RequestProcessingException(msg, ex);
            }
        }
        String outputFormat = request.getParams().getProperty(OUTPUT_PARAM);
        Encoding enc = Encoding.BINARY;
        if (null != outputFormat) {
            try {
                enc = Encoding.valueOf(outputFormat.toUpperCase());
            } catch (Exception ex) {
                LOG.error("Unable to find the output format for bootstrap request for " + outputFormat + ". Using Binary!!", ex);
            }
        }
        processor.setKeyFilter(keyFilter);
        String checkpointString = request.getRequiredStringParam(CHECKPOINT_PARAM);
        int bufferMarginSpace = DEFAULT_BUFFER_MARGIN_SPACE;
        if (null != _serverHostPort) {
            bufferMarginSpace = Math.max(bufferMarginSpace, (_serverHostPort.length() + Checkpoint.BOOTSTRAP_SERVER_INFO.length() + DEFAULT_JSON_OVERHEAD_BYTES));
        }
        int clientFreeBufferSize = request.getRequiredIntParam(BATCHSIZE_PARAM) - checkpointString.length() - bufferMarginSpace;
        BootstrapEventWriter writer = null;
        if (_config.getPredicatePushDown())
            writer = createEventWriter(request, clientFreeBufferSize, null, enc);
        else
            writer = createEventWriter(request, clientFreeBufferSize, keyFilter, enc);
        Checkpoint cp = new Checkpoint(checkpointString);
        DbusClientMode consumptionMode = cp.getConsumptionMode();
        LOG.info("Bootstrap request received: " + "fetchSize=" + clientFreeBufferSize + ", consumptionMode=" + consumptionMode + ", checkpoint=" + checkpointString + ", predicatePushDown=" + _config.getPredicatePushDown());
        try {
            boolean phaseCompleted = false;
            switch(consumptionMode) {
                case BOOTSTRAP_SNAPSHOT:
                    phaseCompleted = processor.streamSnapShotRows(new Checkpoint(checkpointString), writer);
                    break;
                case BOOTSTRAP_CATCHUP:
                    phaseCompleted = processor.streamCatchupRows(new Checkpoint(checkpointString), writer);
                    break;
                default:
                    if (null != bootstrapStatsCollector)
                        bootstrapStatsCollector.registerErrBootstrap();
                    throw new RequestProcessingException("Unexpected mode: " + consumptionMode);
            }
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerBootStrapReq(cp, System.currentTimeMillis() - startTime, clientFreeBufferSize);
            if (writer.getNumRowsWritten() == 0 && writer.getSizeOfPendingEvent() > 0) {
                // Append a header to indicate to the client that we do have at least one event to
                // send, but it is too large to fit into client's offered buffer.
                request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE, writer.getSizeOfPendingEvent());
                if (isDebug) {
                    LOG.debug("Returning 0 events but have pending event of size " + writer.getSizeOfPendingEvent());
                }
            }
            if (phaseCompleted) {
                request.getResponseContent().setMetadata(BootstrapProcessor.PHASE_COMPLETED_HEADER_NAME, BootstrapProcessor.PHASE_COMPLETED_HEADER_TRUE);
            }
        } catch (BootstrapDatabaseTooOldException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrDatabaseTooOld();
            LOG.error("Bootstrap database is too old!", e);
            throw new RequestProcessingException(e);
        } catch (BootstrapDBException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(e);
        } catch (SQLException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrSqlException();
            throw new RequestProcessingException(e);
        } catch (BootstrapProcessingException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(e);
        }
    } finally {
        if (null != processor)
            processor.shutdown();
    }
    return request;
}