Examples with PhysicalPartition com.linkedin.databus.core.data_model.PhysicalPartition used on opensource projects

Example 11 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class TestPhysicalSourceConfig method testPhysicalSourceConfigConstructor.

// test partial constructor
@Test
public void testPhysicalSourceConfigConstructor() {
    Integer pPartitionId = 10;
    String name = "dbName";
    PhysicalPartition pPartition = new PhysicalPartition(pPartitionId, name);
    PhysicalSource pSource = new PhysicalSource("uri");
    PhysicalSourceConfig pConfig = new PhysicalSourceConfig(pPartition.getName(), pSource.getUri(), pPartition.getId());
    int lSourceId = 10;
    String lSourceName = "lName";
    for (int i = 0; i < 10; i++) {
        LogicalSourceConfig lSC = new LogicalSourceConfig();
        lSourceId = lSourceId + i;
        lSC.setId((short) lSourceId);
        lSC.setName(lSourceName + lSourceId);
        lSC.setPartition((short) 0);
        lSC.setUri("lUri");
        pConfig.addSource(lSC);
    }
    assertEquals(10, pConfig.getSources().size(), "number of logical source doesn't match");
}

Example 12 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class TestPhysicalSourceConfig method testParsePhysicalPartitionString.

@Test
public void testParsePhysicalPartitionString() throws IOException {
    String partString = "abc_123";
    PhysicalPartition pPart = PhysicalPartition.parsePhysicalPartitionString(partString, "_");
    assertEquals("abc", pPart.getName());
    assertEquals(123, pPart.getId().intValue());
    partString = "abc.123";
    pPart = PhysicalPartition.parsePhysicalPartitionString(partString, "\\.");
    assertEquals("abc", pPart.getName());
    assertEquals(123, pPart.getId().intValue());
    String[] partStrings = new String[] { "abc.123", "abc123", "123", "abc", "" };
    for (String s : partStrings) {
        try {
            PhysicalPartition.parsePhysicalPartitionString(s, "_");
            fail("should fail on invalid partition string");
        } catch (IOException e) {
        // expected
        }
    }
}

Example 13 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class RelayCommandRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    String command = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME);
    if (null == command) {
        throw new InvalidRequestParamValueException(COMMAND_NAME, "command", "null");
    }
    String reply = "Command " + command + " completed ";
    LOG.info("got relayCommand = " + command);
    if (command.equals(SAVE_META_STATE_PARAM)) {
        _relay.saveBufferMetaInfo(true);
    } else if (command.equals(SHUTDOWN_RELAY_PARAM)) {
        String msg = "received shutdown curl request from: " + request.getRemoteAddress() + ". Shutting down\n";
        LOG.warn(msg);
        request.getResponseContent().write(ByteBuffer.wrap(msg.getBytes("UTF-8")));
        request.getResponseContent().close();
        _relay.shutdown();
    } else if (command.equals(VALIDATE_RELAY_BUFFER_PARAM)) {
        _relay.validateRelayBuffers();
    } else if (command.equals(DISCONNECT_CLIENTS)) {
        Channel rspChannel = request.getResponseContent().getRawChannel();
        _relay.disconnectDBusClients(rspChannel);
    } else if (command.equals(RUN_GC_PARAM)) {
        Runtime rt = Runtime.getRuntime();
        long mem = rt.freeMemory();
        LOG.info("mem before gc = " + rt.freeMemory() + " out of " + rt.totalMemory());
        long time = System.currentTimeMillis();
        System.gc();
        time = System.currentTimeMillis() - time;
        mem = rt.freeMemory() - mem;
        reply = new String("GC run. Took " + time + " millsecs. Freed " + mem + " bytes out of " + rt.totalMemory());
    } else if (command.startsWith(RESET_RELAY_BUFFER_PARAM)) {
        // We expect the request to be of the format:
        //   resetRelayBuffer/<dbName>/<partitionId>?prevScn=<long>&binlogOffset=<long>
        String[] resetCommands = command.split("/");
        if (resetCommands.length != 3) {
            throw new InvalidRequestParamValueException(COMMAND_NAME, "command", command);
        }
        String dbName = resetCommands[1];
        String dbPart = resetCommands[2];
        long prevScn = request.getRequiredLongParam(PREV_SCN_PARAM);
        long binlogOffset = request.getOptionalLongParam(BINLOG_OFFSET_PARAM, 0L);
        LOG.info("reset command = " + dbName + " part =" + dbPart);
        try {
            _relay.resetBuffer(new PhysicalPartition(Integer.parseInt(dbPart), dbName), prevScn, binlogOffset);
        } catch (BufferNotFoundException e) {
            reply = new String("command " + command + ":" + e.getMessage());
        }
    } else if (command.startsWith(GET_BINLOG_OFFSET_PARAM)) {
        String[] getOfsArgs = command.split("/");
        if (getOfsArgs.length != 2) {
            throw new InvalidRequestParamValueException(GET_BINLOG_OFFSET_PARAM, "Server ID", "");
        }
        int serverId;
        try {
            serverId = Integer.parseInt(getOfsArgs[1]);
            int[] offset = _relay.getBinlogOffset(serverId);
            if (offset.length != 2) {
                reply = "Error getting binlog offset";
            } else {
                reply = new String("RelayLastEvent(" + offset[0] + "," + offset[1] + ")");
            }
        } catch (NumberFormatException e) {
            throw new InvalidRequestParamValueException(GET_BINLOG_OFFSET_PARAM, "Server ID", getOfsArgs[1]);
        } catch (DatabusException e) {
            reply = new String("command " + command + "failed with:" + e.getMessage());
        }
    } else if (command.startsWith(PRINT_RELAY_INFO_PARAM)) {
        try {
            Map<String, String> infoMap = _relay.printInfo();
            reply = makeJsonResponse(infoMap, request);
        } catch (Exception e) {
            reply = new String("command " + command + " failed with:" + e.getMessage());
        }
    } else {
        // invalid command
        reply = new String("command " + command + " is invalid. Valid commands are: " + SAVE_META_STATE_PARAM + "|" + SHUTDOWN_RELAY_PARAM + "|" + VALIDATE_RELAY_BUFFER_PARAM + "|" + RUN_GC_PARAM + "|" + RESET_RELAY_BUFFER_PARAM + "|" + GET_BINLOG_OFFSET_PARAM + "|" + PRINT_RELAY_INFO_PARAM + "|" + DISCONNECT_CLIENTS);
    }
    byte[] responseBytes = new byte[(reply.length() + 2)];
    System.arraycopy(reply.getBytes("UTF-8"), 0, responseBytes, 0, reply.length());
    int idx = reply.length();
    responseBytes[idx] = (byte) '\r';
    responseBytes[idx + 1] = (byte) '\n';
    request.getResponseContent().write(ByteBuffer.wrap(responseBytes));
    return request;
}

Example 14 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class HttpRelay method dropDatabase.

@Override
public void dropDatabase(String dbName) throws DatabusException {
    _schemaRegistryService.dropDatabase(dbName);
    DbusEventBufferMult eventMult = getEventBuffer();
    /*
    	 * Close the buffers
    	 */
    for (DbusEventBuffer dBuf : eventMult.bufIterable()) {
        PhysicalPartition pp = dBuf.getPhysicalPartition();
        if (pp.getName().equals(dbName)) {
            dBuf.closeBuffer(false);
            dBuf.removeMMapFiles();
            PhysicalPartitionKey pKey = new PhysicalPartitionKey(pp);
            eventMult.removeBuffer(pKey, null);
        }
    }
    eventMult.deallocateRemovedBuffers(true);
    return;
}

Example 15 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class BufferInfoRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException, DatabusException {
    String category = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME);
    if (null == category) {
        throw new InvalidRequestParamValueException(COMMAND_NAME, "category", "null");
    }
    if (category.startsWith(INBOUND_VIEW)) {
        String sourceIdStr = category.substring(INBOUND_VIEW.length());
        sourceIdStr = sourceIdStr.replace('/', ':');
        PhysicalPartition pPartition = PhysicalPartition.parsePhysicalPartitionString(sourceIdStr, ":");
        processInboundRequest(request, pPartition);
    } else {
        throw new InvalidRequestParamValueException(COMMAND_NAME, "category", category);
    }
    return request;
}