Examples with DatabusRequest com.linkedin.databus2.core.container.request.DatabusRequest used on opensource projects

Example 11 with DatabusRequest

use of com.linkedin.databus2.core.container.request.DatabusRequest in project databus by linkedin.

the class ContainerStatsRequestProcessor method processOutboundTrafficClientStats.

private void processOutboundTrafficClientStats(DatabusRequest request) throws IOException, RequestProcessingException {
    String category = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME);
    String prefix = "outbound/client/";
    String client = category.substring(prefix.length());
    ContainerTrafficTotalStats clientStats = _containerStatsCollector.getOutboundClientStats(client);
    if (null == clientStats) {
        throw new InvalidRequestParamValueException(request.getName(), prefix, client);
    }
    JsonEncoder jsonEncoder = clientStats.createJsonEncoder(Channels.newOutputStream(request.getResponseContent()));
    clientStats.toJson(jsonEncoder, null);
    if (request.getRequestType() == HttpMethod.PUT || request.getRequestType() == HttpMethod.POST) {
        enableOrResetStatsMBean(clientStats, request);
    }
}

Example 12 with DatabusRequest

use of com.linkedin.databus2.core.container.request.DatabusRequest in project databus by linkedin.

the class ContainerStatsRequestProcessor method processOutboundTrafficTotalStats.

private void processOutboundTrafficTotalStats(DatabusRequest request) throws IOException {
    ContainerTrafficTotalStatsMBean outboundTrafficTotalStatsMBean = _containerStatsCollector.getOutboundTrafficTotalStats();
    if (null == outboundTrafficTotalStatsMBean)
        return;
    //String json = outboundTrafficTotalStatsMBean.toJson();
    JsonEncoder jsonEncoder = outboundTrafficTotalStatsMBean.createJsonEncoder(Channels.newOutputStream(request.getResponseContent()));
    outboundTrafficTotalStatsMBean.toJson(jsonEncoder, null);
    if (request.getRequestType() == HttpMethod.PUT || request.getRequestType() == HttpMethod.POST) {
        enableOrResetStatsMBean(outboundTrafficTotalStatsMBean, request);
    }
}

Example 13 with DatabusRequest

use of com.linkedin.databus2.core.container.request.DatabusRequest in project databus by linkedin.

the class RegisterRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    try {
        // fail early if optional version param is included but isn't valid
        // 2 and 3 are same for us; 4 is a superset only newer clients understand
        int registerRequestProtocolVersion = 3;
        String registerRequestProtocolVersionStr = request.getParams().getProperty(DatabusHttpHeaders.PROTOCOL_VERSION_PARAM);
        if (registerRequestProtocolVersionStr != null) {
            try {
                registerRequestProtocolVersion = Integer.parseInt(registerRequestProtocolVersionStr);
            } catch (NumberFormatException e) {
                LOG.error("Could not parse /register request protocol version: " + registerRequestProtocolVersionStr);
                throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, registerRequestProtocolVersionStr);
            }
            if (registerRequestProtocolVersion < 2 || registerRequestProtocolVersion > 4) {
                LOG.error("Out-of-range /register request protocol version: " + registerRequestProtocolVersionStr);
                throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, registerRequestProtocolVersionStr);
            }
        }
        Collection<LogicalSource> logicalSources = null;
        HttpStatisticsCollector relayStatsCollector = _relay.getHttpStatisticsCollector();
        String sources = request.getParams().getProperty(SOURCES_PARAM);
        if (null == sources) {
            // need to return all schemas, so first get all sources
            logicalSources = _relay.getSourcesIdNameRegistry().getAllSources();
        } else {
            String[] sourceIds = sources.split(",");
            logicalSources = new ArrayList<LogicalSource>(sourceIds.length);
            for (String sourceId : sourceIds) {
                int srcId;
                String trimmedSourceId = sourceId.trim();
                try {
                    srcId = Integer.valueOf(trimmedSourceId);
                    LogicalSource lsource = _relay.getSourcesIdNameRegistry().getSource(srcId);
                    if (null != lsource)
                        logicalSources.add(lsource);
                    else {
                        LOG.error("No source name for source id: " + srcId);
                        throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
                    }
                } catch (NumberFormatException nfe) {
                    if (relayStatsCollector != null) {
                        relayStatsCollector.registerInvalidRegisterCall();
                    }
                    throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
                }
            }
        }
        SchemaRegistryService schemaRegistry = _relay.getSchemaRegistryService();
        ArrayList<RegisterResponseEntry> registeredSources = new ArrayList<RegisterResponseEntry>(20);
        for (LogicalSource lsource : logicalSources) {
            getSchemas(schemaRegistry, lsource.getName(), lsource.getId(), sources, registeredSources);
        }
        // Note that, as of April 2013, the Espresso sandbox's schema registry
        // (in JSON format) is 4.5 MB and growing.  But 100 KB is probably OK
        // for regular production cases.
        StringWriter out = new StringWriter(102400);
        ObjectMapper mapper = new ObjectMapper();
        // any circumstances under which we might want to override this?
        int registerResponseProtocolVersion = registerRequestProtocolVersion;
        if (// DDSDBUS-2009
        registerRequestProtocolVersion == 4) {
            LOG.debug("Got version 4 /register request; fetching metadata schema.");
            // Get (replication) metadata schema from registry; format it as list
            // of schemas (multiple only if more than one version exists).  Per
            // https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Espresso+Metadata+Schema,
            // name of replication metadata is simply "metadata".
            ArrayList<RegisterResponseMetadataEntry> registeredMetadata = new ArrayList<RegisterResponseMetadataEntry>(2);
            getMetadataSchemas(schemaRegistry, registeredMetadata);
            // Set up the v4 response as a map:  one entry is the existing list of source
            // schemas, and the others (if present) are the new lists of metadata schema(s)
            // and (TODO) key schemas.
            HashMap<String, List<Object>> responseMap = new HashMap<String, List<Object>>(4);
            responseMap.put(RegisterResponseEntry.SOURCE_SCHEMAS_KEY, (List<Object>) (List<?>) registeredSources);
            if (registeredMetadata.size() > 0) {
                LOG.debug("Sending v4 /register response with metadata schema.");
                responseMap.put(RegisterResponseMetadataEntry.METADATA_SCHEMAS_KEY, (List<Object>) (List<?>) registeredMetadata);
            } else {
                LOG.debug("No metadata schema available; sending v4 /register response without.");
            }
            // TODO:  figure out how to retrieve key schemas and include via RegisterResponseEntry.KEY_SCHEMAS_KEY
            mapper.writeValue(out, responseMap);
        } else // fall back to old style (v2/v3 response)
        {
            mapper.writeValue(out, registeredSources);
        }
        ChunkedWritableByteChannel responseContent = request.getResponseContent();
        byte[] resultBytes = out.toString().getBytes(Charset.defaultCharset());
        responseContent.addMetadata(DatabusHttpHeaders.DBUS_CLIENT_RELAY_PROTOCOL_VERSION_HDR, registerResponseProtocolVersion);
        responseContent.write(ByteBuffer.wrap(resultBytes));
        if (null != relayStatsCollector) {
            HttpStatisticsCollector connStatsCollector = (HttpStatisticsCollector) request.getParams().get(relayStatsCollector.getName());
            if (null != connStatsCollector) {
                connStatsCollector.registerRegisterCall(registeredSources);
            } else {
                relayStatsCollector.registerRegisterCall(registeredSources);
            }
        }
        return request;
    } catch (InvalidRequestParamValueException e) {
        HttpStatisticsCollector relayStatsCollector = _relay.getHttpStatisticsCollector();
        if (null != relayStatsCollector)
            relayStatsCollector.registerInvalidRegisterCall();
        throw e;
    }
}

Example 14 with DatabusRequest

use of com.linkedin.databus2.core.container.request.DatabusRequest in project databus by linkedin.

the class RelayCommandRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    String command = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME);
    if (null == command) {
        throw new InvalidRequestParamValueException(COMMAND_NAME, "command", "null");
    }
    String reply = "Command " + command + " completed ";
    LOG.info("got relayCommand = " + command);
    if (command.equals(SAVE_META_STATE_PARAM)) {
        _relay.saveBufferMetaInfo(true);
    } else if (command.equals(SHUTDOWN_RELAY_PARAM)) {
        String msg = "received shutdown curl request from: " + request.getRemoteAddress() + ". Shutting down\n";
        LOG.warn(msg);
        request.getResponseContent().write(ByteBuffer.wrap(msg.getBytes("UTF-8")));
        request.getResponseContent().close();
        _relay.shutdown();
    } else if (command.equals(VALIDATE_RELAY_BUFFER_PARAM)) {
        _relay.validateRelayBuffers();
    } else if (command.equals(DISCONNECT_CLIENTS)) {
        Channel rspChannel = request.getResponseContent().getRawChannel();
        _relay.disconnectDBusClients(rspChannel);
    } else if (command.equals(RUN_GC_PARAM)) {
        Runtime rt = Runtime.getRuntime();
        long mem = rt.freeMemory();
        LOG.info("mem before gc = " + rt.freeMemory() + " out of " + rt.totalMemory());
        long time = System.currentTimeMillis();
        System.gc();
        time = System.currentTimeMillis() - time;
        mem = rt.freeMemory() - mem;
        reply = new String("GC run. Took " + time + " millsecs. Freed " + mem + " bytes out of " + rt.totalMemory());
    } else if (command.startsWith(RESET_RELAY_BUFFER_PARAM)) {
        // We expect the request to be of the format:
        //   resetRelayBuffer/<dbName>/<partitionId>?prevScn=<long>&binlogOffset=<long>
        String[] resetCommands = command.split("/");
        if (resetCommands.length != 3) {
            throw new InvalidRequestParamValueException(COMMAND_NAME, "command", command);
        }
        String dbName = resetCommands[1];
        String dbPart = resetCommands[2];
        long prevScn = request.getRequiredLongParam(PREV_SCN_PARAM);
        long binlogOffset = request.getOptionalLongParam(BINLOG_OFFSET_PARAM, 0L);
        LOG.info("reset command = " + dbName + " part =" + dbPart);
        try {
            _relay.resetBuffer(new PhysicalPartition(Integer.parseInt(dbPart), dbName), prevScn, binlogOffset);
        } catch (BufferNotFoundException e) {
            reply = new String("command " + command + ":" + e.getMessage());
        }
    } else if (command.startsWith(GET_BINLOG_OFFSET_PARAM)) {
        String[] getOfsArgs = command.split("/");
        if (getOfsArgs.length != 2) {
            throw new InvalidRequestParamValueException(GET_BINLOG_OFFSET_PARAM, "Server ID", "");
        }
        int serverId;
        try {
            serverId = Integer.parseInt(getOfsArgs[1]);
            int[] offset = _relay.getBinlogOffset(serverId);
            if (offset.length != 2) {
                reply = "Error getting binlog offset";
            } else {
                reply = new String("RelayLastEvent(" + offset[0] + "," + offset[1] + ")");
            }
        } catch (NumberFormatException e) {
            throw new InvalidRequestParamValueException(GET_BINLOG_OFFSET_PARAM, "Server ID", getOfsArgs[1]);
        } catch (DatabusException e) {
            reply = new String("command " + command + "failed with:" + e.getMessage());
        }
    } else if (command.startsWith(PRINT_RELAY_INFO_PARAM)) {
        try {
            Map<String, String> infoMap = _relay.printInfo();
            reply = makeJsonResponse(infoMap, request);
        } catch (Exception e) {
            reply = new String("command " + command + " failed with:" + e.getMessage());
        }
    } else {
        // invalid command
        reply = new String("command " + command + " is invalid. Valid commands are: " + SAVE_META_STATE_PARAM + "|" + SHUTDOWN_RELAY_PARAM + "|" + VALIDATE_RELAY_BUFFER_PARAM + "|" + RUN_GC_PARAM + "|" + RESET_RELAY_BUFFER_PARAM + "|" + GET_BINLOG_OFFSET_PARAM + "|" + PRINT_RELAY_INFO_PARAM + "|" + DISCONNECT_CLIENTS);
    }
    byte[] responseBytes = new byte[(reply.length() + 2)];
    System.arraycopy(reply.getBytes("UTF-8"), 0, responseBytes, 0, reply.length());
    int idx = reply.length();
    responseBytes[idx] = (byte) '\r';
    responseBytes[idx + 1] = (byte) '\n';
    request.getResponseContent().write(ByteBuffer.wrap(responseBytes));
    return request;
}

Example 15 with DatabusRequest

use of com.linkedin.databus2.core.container.request.DatabusRequest in project databus by linkedin.

the class RelayStatsRequestProcessor method processInboundGGStats.

private void processInboundGGStats(DatabusRequest request, String category) throws IOException, RequestProcessingException {
    if (!(_relay instanceof DatabusRelayMain)) {
        throw new IllegalArgumentException(category + " for relay which is not DatabusRelayMain");
    }
    String psourceName = null;
    if (category.startsWith(INBOUND_GG_PSOURCE_PREFIX)) {
        psourceName = category.substring(INBOUND_GG_PSOURCE_PREFIX.length());
        if (psourceName == null || psourceName.length() <= 0) {
            throw new InvalidRequestParamValueException(request.getName(), INBOUND_GG_PSOURCE_PREFIX, null);
        }
        LOG.info("get parser stats for source " + psourceName);
    }
    List<String> phSourceNames = new ArrayList<String>();
    EventProducer[] prods = ((DatabusRelayMain) _relay).getProducers();
    GGParserStatistics stat = null;
    for (EventProducer prod : prods) {
        if (prod != null && (prod instanceof GoldenGateEventProducer)) {
            GoldenGateEventProducer ggProducer = (GoldenGateEventProducer) prod;
            String pSrcName = ggProducer.getParserStats().getPhysicalSourceName();
            phSourceNames.add(pSrcName);
            if (// remember the stats object
            psourceName != null && psourceName.equals(pSrcName))
                stat = ggProducer.getParserStats();
        }
    }
    if (psourceName != null) {
        if (stat == null)
            throw new InvalidRequestParamValueException(request.getName(), INBOUND_GG_PSOURCE_PREFIX, psourceName);
        writeJsonObjectToResponse(stat, request);
    } else {
        writeJsonObjectToResponse(phSourceNames, request);
    }
}