Examples with IStatisticManager org.jumpmind.symmetric.statistic.IStatisticManager used on opensource projects

Example 1 with IStatisticManager

use of org.jumpmind.symmetric.statistic.IStatisticManager in project symmetric-ds by JumpMind.

the class PushUriHandler method handle.

public void handle(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
    String nodeId = ServletUtils.getParameter(req, WebConstants.NODE_ID);
    String channelId = getChannelId(req);
    log.info("About to service push request for {}", nodeId);
    IStagingManager stagingManager = engine.getStagingManager();
    IDataLoaderService dataLoaderService = engine.getDataLoaderService();
    INodeService nodeService = engine.getNodeService();
    IStatisticManager statisticManager = engine.getStatisticManager();
    String identityNodeId = nodeService.findIdentityNodeId();
    ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(nodeId, identityNodeId, ProcessType.TRANSFER_FROM, channelId));
    BufferedReader reader = null;
    BufferedWriter writer = null;
    DataLoaderWorker worker = null;
    try {
        Node sourceNode = engine.getNodeService().findNode(nodeId);
        processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
        reader = new BufferedReader(new InputStreamReader(createInputStream(req)));
        long streamToFileThreshold = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD);
        String line = reader.readLine();
        StringBuilder batchPrefix = new StringBuilder();
        Long batchId = null;
        while (line != null) {
            if (line.startsWith(CsvConstants.BATCH)) {
                batchId = getBatchId(line);
                IStagedResource resource = stagingManager.create(streamToFileThreshold, Constants.STAGING_CATEGORY_INCOMING, nodeId, batchId);
                writer = resource.getWriter();
                writer.write(batchPrefix.toString());
            } else if (line.startsWith(CsvConstants.COMMIT)) {
                writer.write(line);
                writer.close();
                writer = null;
                if (worker == null) {
                    worker = dataLoaderService.createDataLoaderWorker(ProcessType.LOAD_FROM_PUSH, channelId, sourceNode);
                }
                worker.queueUpLoad(new IncomingBatch(batchId, nodeId));
                batchId = null;
            }
            if (batchId == null) {
                batchPrefix.append(line).append("\n");
            } else if (writer != null) {
                writer.write(line);
                writer.write("\n");
            }
            line = reader.readLine();
        }
        processInfo.setStatus(ProcessInfo.Status.OK);
    } catch (RuntimeException ex) {
        processInfo.setStatus(ProcessInfo.Status.ERROR);
        throw ex;
    } finally {
        IOUtils.closeQuietly(reader);
        IOUtils.closeQuietly(writer);
    }
    PrintWriter resWriter = res.getWriter();
    if (worker != null) {
        worker.queueUpLoad(new DataLoaderService.EOM());
        while (!worker.isComplete()) {
            String status = "done";
            IncomingBatch batch = worker.waitForNextBatchToComplete();
            if (batch == null) {
                status = "in progress";
                batch = worker.getCurrentlyLoading();
            }
            if (batch != null && !(batch instanceof DataLoaderService.EOM)) {
                ArrayList<IncomingBatch> list = new ArrayList<IncomingBatch>(1);
                list.add(batch);
                log.info("sending {} ack ... for {}", status, batch);
                // TODO 13 support
                resWriter.write(engine.getTransportManager().getAcknowledgementData(false, identityNodeId, list));
                resWriter.write("\n");
                resWriter.flush();
            }
        }
    }
    res.flushBuffer();
    log.debug("Done servicing push request for {}", nodeId);
}

Example 2 with IStatisticManager

use of org.jumpmind.symmetric.statistic.IStatisticManager in project symmetric-ds by JumpMind.

the class SimpleIntegrationTest method test02RegisterClientWithRoot.

@Test(timeout = 240000)
public void test02RegisterClientWithRoot() {
    logTestRunning();
    ISymmetricEngine rootEngine = getServer();
    INodeService rootNodeService = rootEngine.getNodeService();
    rootEngine.openRegistration(TestConstants.TEST_CLIENT_NODE_GROUP, TestConstants.TEST_CLIENT_EXTERNAL_ID);
    assertTrue("The registration for the client should be opened now", rootNodeService.findNodeSecurity(TestConstants.TEST_CLIENT_EXTERNAL_ID).isRegistrationEnabled());
    getClient().start();
    getClient().getParameterService().saveParameter(ParameterConstants.FILE_SYNC_ENABLE, false, "unit_test");
    clientPull();
    assertTrue("The client did not register", getClient().isRegistered());
    assertFalse("The registration for the client should be closed now", rootNodeService.findNodeSecurity(TestConstants.TEST_CLIENT_EXTERNAL_ID).isRegistrationEnabled());
    IStatisticManager statMgr = getClient().getStatisticManager();
    statMgr.flush();
    checkForFailedTriggers(true, true);
}

Example 3 with IStatisticManager

use of org.jumpmind.symmetric.statistic.IStatisticManager in project symmetric-ds by JumpMind.

the class DataGapRouteReaderTest method buildReader.

protected DataGapRouteReader buildReader(int peekAheadMemoryThreshold, List<DataGap> dataGaps) throws Exception {
    when(parameterService.getEngineName()).thenReturn(ENGINE_NAME);
    when(parameterService.is(ParameterConstants.SYNCHRONIZE_ALL_JOBS)).thenReturn(true);
    when(parameterService.getInt(ParameterConstants.ROUTING_WAIT_FOR_DATA_TIMEOUT_SECONDS)).thenReturn(330);
    when(parameterService.getInt(ParameterConstants.ROUTING_PEEK_AHEAD_MEMORY_THRESHOLD)).thenReturn(peekAheadMemoryThreshold);
    when(parameterService.getInt(ParameterConstants.ROUTING_MAX_GAPS_TO_QUALIFY_IN_SQL)).thenReturn(100);
    when(parameterService.getInt(ParameterConstants.ROUTING_DATA_READER_THRESHOLD_GAPS_TO_USE_GREATER_QUERY)).thenReturn(100);
    when(parameterService.is(ParameterConstants.ROUTING_DATA_READER_ORDER_BY_DATA_ID_ENABLED)).thenReturn(true);
    IStatisticManager statisticManager = mock(StatisticManager.class);
    when(statisticManager.newProcessInfo((ProcessInfoKey) any())).thenReturn(new ProcessInfo());
    INodeService nodeService = mock(NodeService.class);
    when(nodeService.findIdentity()).thenReturn(new Node(NODE_ID, NODE_GROUP_ID));
    IDatabasePlatform platform = mock(IDatabasePlatform.class);
    when(platform.getSqlTemplate()).thenReturn(sqlTemplate);
    when(platform.getDatabaseInfo()).thenReturn(new DatabaseInfo());
    ISymmetricDialect symmetricDialect = mock(AbstractSymmetricDialect.class);
    when(symmetricDialect.supportsTransactionId()).thenReturn(true);
    when(symmetricDialect.getPlatform()).thenReturn(platform);
    IExtensionService extensionService = mock(ExtensionService.class);
    ISymmetricEngine engine = mock(AbstractSymmetricEngine.class);
    when(engine.getParameterService()).thenReturn(parameterService);
    when(engine.getStatisticManager()).thenReturn(statisticManager);
    when(engine.getNodeService()).thenReturn(nodeService);
    when(engine.getDataService()).thenReturn(dataService);
    when(engine.getSymmetricDialect()).thenReturn(symmetricDialect);
    when(engine.getExtensionService()).thenReturn(extensionService);
    IRouterService routerService = new RouterService(engine);
    when(engine.getRouterService()).thenReturn(routerService);
    ChannelRouterContext context = new ChannelRouterContext(NODE_ID, nodeChannel, mock(ISqlTransaction.class));
    context.setDataGaps(dataGaps);
    return new DataGapRouteReader(context, engine);
}

Example 4 with IStatisticManager

use of org.jumpmind.symmetric.statistic.IStatisticManager in project symmetric-ds by JumpMind.

the class DataGapRouteReaderTest method buildReader.

protected DataGapRouteReader buildReader(int peekAheadMemoryThreshold) throws Exception {
    when(parameterService.getEngineName()).thenReturn(ENGINE_NAME);
    when(parameterService.is(ParameterConstants.SYNCHRONIZE_ALL_JOBS)).thenReturn(true);
    when(parameterService.getInt(ParameterConstants.ROUTING_WAIT_FOR_DATA_TIMEOUT_SECONDS)).thenReturn(330);
    when(parameterService.getInt(ParameterConstants.ROUTING_PEEK_AHEAD_MEMORY_THRESHOLD)).thenReturn(peekAheadMemoryThreshold);
    when(parameterService.getInt(ParameterConstants.ROUTING_MAX_GAPS_TO_QUALIFY_IN_SQL)).thenReturn(100);
    when(parameterService.getInt(ParameterConstants.ROUTING_DATA_READER_THRESHOLD_GAPS_TO_USE_GREATER_QUERY)).thenReturn(100);
    when(parameterService.is(ParameterConstants.ROUTING_DATA_READER_ORDER_BY_DATA_ID_ENABLED)).thenReturn(true);
    IStatisticManager statisticManager = mock(StatisticManager.class);
    when(statisticManager.newProcessInfo((ProcessInfoKey) any())).thenReturn(new ProcessInfo());
    INodeService nodeService = mock(NodeService.class);
    when(nodeService.findIdentity()).thenReturn(new Node(NODE_ID, NODE_GROUP_ID));
    IDatabasePlatform platform = mock(IDatabasePlatform.class);
    when(platform.getSqlTemplate()).thenReturn(sqlTemplate);
    when(platform.getDatabaseInfo()).thenReturn(new DatabaseInfo());
    ISymmetricDialect symmetricDialect = mock(AbstractSymmetricDialect.class);
    when(symmetricDialect.supportsTransactionId()).thenReturn(true);
    when(symmetricDialect.getPlatform()).thenReturn(platform);
    IExtensionService extensionService = mock(ExtensionService.class);
    ISymmetricEngine engine = mock(AbstractSymmetricEngine.class);
    when(engine.getParameterService()).thenReturn(parameterService);
    when(engine.getStatisticManager()).thenReturn(statisticManager);
    when(engine.getNodeService()).thenReturn(nodeService);
    when(engine.getDataService()).thenReturn(dataService);
    when(engine.getSymmetricDialect()).thenReturn(symmetricDialect);
    when(engine.getExtensionService()).thenReturn(extensionService);
    IRouterService routerService = new RouterService(engine);
    when(engine.getRouterService()).thenReturn(routerService);
    ChannelRouterContext context = new ChannelRouterContext(NODE_ID, nodeChannel, mock(ISqlTransaction.class));
    return new DataGapRouteReader(context, engine);
}

Example 5 with IStatisticManager

use of org.jumpmind.symmetric.statistic.IStatisticManager in project symmetric-ds by JumpMind.

the class RestService method getPullData.

@ApiOperation(value = "Pull pending batches for the specified node for the specified engine")
@RequestMapping(value = "/engine/{engine}/pulldata", method = RequestMethod.GET)
@ResponseStatus(HttpStatus.OK)
@ResponseBody
public final PullDataResults getPullData(@PathVariable("engine") String engineName, @RequestParam(value = WebConstants.NODE_ID) String nodeId, @ApiParam(value = "This the password for the nodeId being passed in.  The password is stored in the node_security table.") @RequestParam(value = WebConstants.SECURITY_TOKEN) String securityToken, @RequestParam(value = "useJdbcTimestampFormat", required = false, defaultValue = "true") boolean useJdbcTimestampFormat, @RequestParam(value = "useUpsertStatements", required = false, defaultValue = "false") boolean useUpsertStatements, @RequestParam(value = "useDelimitedIdentifiers", required = false, defaultValue = "true") boolean useDelimitedIdentifiers, @RequestParam(value = "hostName", required = false) String hostName) {
    ISymmetricEngine engine = getSymmetricEngine(engineName);
    IDataExtractorService dataExtractorService = engine.getDataExtractorService();
    IStatisticManager statisticManager = engine.getStatisticManager();
    INodeService nodeService = engine.getNodeService();
    org.jumpmind.symmetric.model.Node targetNode = nodeService.findNode(nodeId);
    if (securityVerified(nodeId, engine, securityToken)) {
        ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(nodeService.findIdentityNodeId(), nodeId, ProcessType.REST_PULL_HANLDER));
        try {
            PullDataResults results = new PullDataResults();
            List<OutgoingBatchWithPayload> extractedBatches = dataExtractorService.extractToPayload(processInfo, targetNode, PayloadType.SQL, useJdbcTimestampFormat, useUpsertStatements, useDelimitedIdentifiers);
            List<Batch> batches = new ArrayList<Batch>();
            for (OutgoingBatchWithPayload outgoingBatchWithPayload : extractedBatches) {
                if (outgoingBatchWithPayload.getStatus() == org.jumpmind.symmetric.model.OutgoingBatch.Status.LD || outgoingBatchWithPayload.getStatus() == org.jumpmind.symmetric.model.OutgoingBatch.Status.IG) {
                    Batch batch = new Batch();
                    batch.setBatchId(outgoingBatchWithPayload.getBatchId());
                    batch.setChannelId(outgoingBatchWithPayload.getChannelId());
                    batch.setSqlStatements(outgoingBatchWithPayload.getPayload());
                    batches.add(batch);
                }
            }
            results.setBatches(batches);
            results.setNbrBatches(batches.size());
            processInfo.setStatus(org.jumpmind.symmetric.model.ProcessInfo.Status.OK);
            if (engine.getParameterService().is(ParameterConstants.REST_HEARTBEAT_ON_PULL) && hostName != null) {
                Heartbeat heartbeat = new Heartbeat();
                heartbeat.setNodeId(nodeId);
                heartbeat.setHeartbeatTime(new Date());
                heartbeat.setHostName(hostName);
                this.heartbeatImpl(engine, heartbeat);
            }
            return results;
        } finally {
            if (processInfo.getStatus() != org.jumpmind.symmetric.model.ProcessInfo.Status.OK) {
                processInfo.setStatus(org.jumpmind.symmetric.model.ProcessInfo.Status.ERROR);
            }
        }
    } else {
        throw new NotAllowedException();
    }
}