Examples with ReplicationState org.openstreetmap.osmosis.replication.common.ReplicationState used on opensource projects

Example 1 with ReplicationState

use of org.openstreetmap.osmosis.replication.common.ReplicationState in project osmosis by openstreetmap.

the class MockReplicationSource method sendSequence.

/**
 * Sends a replication sequence containing dummy data to the destination.
 */
public void sendSequence() {
    // Initialise the replication stream.
    ReplicationState state = new ReplicationState();
    Map<String, Object> metaData = new HashMap<String, Object>(1);
    metaData.put(ReplicationState.META_DATA_KEY, state);
    changeSink.initialize(metaData);
    // allowed. We'll only send a single record for simplicity.
    if (state.getSequenceNumber() > 0) {
        // We'll do a create action on the first replication pass, and modify subsequently.
        ChangeAction action;
        if (state.getSequenceNumber() == 1) {
            action = ChangeAction.Create;
        } else {
            action = ChangeAction.Modify;
        }
        // Create a change record which data derived from the
        // replication sequence number itself.
        ChangeContainer change = new ChangeContainer(new NodeContainer(new Node(new CommonEntityData(10, (int) state.getSequenceNumber(), new Date(state.getSequenceNumber() * 1000), new OsmUser(11, "test"), state.getSequenceNumber() * 2), state.getSequenceNumber() * 3, state.getSequenceNumber() * 4)), action);
        // Send the record downstream.
        changeSink.process(change);
    }
    state.setTimestamp(new Date(state.getSequenceNumber() * 1000));
    changeSink.complete();
}

Example 2 with ReplicationState

use of org.openstreetmap.osmosis.replication.common.ReplicationState in project osmosis by openstreetmap.

the class BaseReplicationDownloader method download.

private ReplicationState download(ReplicationDownloaderConfiguration configuration, ReplicationState serverState, ReplicationState initialLocalState) {
    URL baseUrl;
    ReplicationState localState;
    Date maximumDownloadTimestamp;
    localState = initialLocalState;
    // Determine the location of download files.
    baseUrl = configuration.getBaseUrl();
    // Determine the maximum timestamp that can be downloaded.
    maximumDownloadTimestamp = calculateMaximumTimestamp(configuration, serverState.getTimestamp(), localState.getTimestamp());
    LOG.fine("The maximum timestamp to be downloaded is " + maximumDownloadTimestamp + ".");
    // Download all files and send their contents to the sink.
    while (localState.getSequenceNumber() < serverState.getSequenceNumber()) {
        File replicationFile;
        long sequenceNumber;
        ReplicationState fileReplicationState;
        // time period of data to become available before processing.
        if (localState.getTimestamp().compareTo(maximumDownloadTimestamp) >= 0) {
            break;
        }
        // Calculate the next sequence number.
        sequenceNumber = localState.getSequenceNumber() + 1;
        LOG.finer("Processing replication sequence " + sequenceNumber + ".");
        // Get the state associated with the next file.
        fileReplicationState = serverStateReader.getServerState(baseUrl, sequenceNumber);
        // has been reached.
        if (fileReplicationState.getTimestamp().compareTo(maximumDownloadTimestamp) > 0) {
            // a long time gap between two intervals due to system downtime.
            if (localState.getSequenceNumber() != initialLocalState.getSequenceNumber()) {
                break;
            }
        }
        // Download the next replication file to a temporary file.
        replicationFile = downloadReplicationFile(sequenceFormatter.getFormattedName(sequenceNumber, ".osc.gz"), baseUrl);
        // Process the file and send its contents to the sink.
        processReplicationFile(replicationFile, fileReplicationState);
        // Update the local state to reflect the file state just processed.
        localState = fileReplicationState;
        // and not up to the current one.
        if (single) {
            break;
        }
    }
    return localState;
}

Example 3 with ReplicationState

use of org.openstreetmap.osmosis.replication.common.ReplicationState in project osmosis by openstreetmap.

the class BaseReplicationDownloader method runImpl.

private void runImpl() {
    try {
        ReplicationDownloaderConfiguration configuration;
        ReplicationState serverState;
        ReplicationState localState;
        PropertiesPersister localStatePersistor;
        // Instantiate utility objects.
        configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));
        // check for custom server state file
        File customServerStateFile = new File(workingDirectory, CUSTOM_SERVER_STATE_FILE);
        if (customServerStateFile.exists()) {
            serverState = new ReplicationState(new PropertiesPersister(customServerStateFile).loadMap());
            LOG.info(String.format("Reading custom server state. [%s]", serverState.toString()));
        } else {
            // Obtain the server state.
            serverState = serverStateReader.getServerState(configuration.getBaseUrl());
            LOG.info(String.format("Reading current server state. [%s]", serverState.toString()));
        }
        // Obtain the server state.
        LOG.fine("Reading current server state.");
        serverState = serverStateReader.getServerState(configuration.getBaseUrl());
        // Build the local state persister which is used for both loading and storing local state.
        localStatePersistor = new PropertiesPersister(new File(workingDirectory, LOCAL_STATE_FILE));
        // Begin processing.
        processInitialize(Collections.<String, Object>emptyMap());
        // then exit.
        if (localStatePersistor.exists()) {
            localState = new ReplicationState(localStatePersistor.loadMap());
            // Download and process the replication files.
            localState = download(configuration, serverState, localState);
        } else {
            localState = serverState;
            processInitializeState(localState);
        }
        // Commit downstream changes.
        processComplete();
        // Persist the local state.
        localStatePersistor.store(localState.store());
    } finally {
        processRelease();
    }
}

Example 4 with ReplicationState

use of org.openstreetmap.osmosis.replication.common.ReplicationState in project osmosis by openstreetmap.

the class ReplicationDataClientHandler method processMessageData.

@Override
protected void processMessageData(ChannelBuffer buffer) {
    // Break the data down according to chunk alignment.
    List<File> chunkFiles = chunkReceiver.processData(buffer);
    try {
        for (File chunkFile : chunkFiles) {
            if (!replicationStateReceived) {
                // while preparing our initial request.
                if (!sinkInitInvoked) {
                    invokeSinkInit();
                }
                // The first chunk contains the replication state stored in
                // properties format.
                ReplicationState serverReplicationState = loadState(chunkFile);
                if (LOG.isLoggable(Level.FINER)) {
                    LOG.finer("Received replication state " + serverReplicationState.getSequenceNumber());
                }
                // Validate that the server has sent us the expected state.
                if (serverReplicationState.getSequenceNumber() != replicationState.getSequenceNumber()) {
                    throw new OsmosisRuntimeException("Received sequence number " + serverReplicationState.getSequenceNumber() + " from server, expected " + replicationState.getSequenceNumber());
                }
                // Update the local state with server values.
                replicationState.setTimestamp(serverReplicationState.getTimestamp());
                replicationStateReceived = true;
                // any data.
                if (replicationState.getSequenceNumber() == 0) {
                    sendReplicationData(null);
                }
            } else {
                sendReplicationData(chunkFile);
            }
        }
    } finally {
        // Delete all chunk files.
        for (File chunkFile : chunkFiles) {
            if (!chunkFile.delete()) {
                LOG.log(Level.WARNING, "Unable to delete the current temporary chunk file " + chunkFile);
            }
        }
    }
}

Example 5 with ReplicationState

use of org.openstreetmap.osmosis.replication.common.ReplicationState in project osmosis by openstreetmap.

the class ReplicationFileMerger method processInitializeState.

/**
 * {@inheritDoc}
 */
@Override
protected void processInitializeState(ReplicationState initialState) {
    Date initialDate;
    Date alignedDate;
    long intervalLength;
    intervalLength = getConfiguration().getIntervalLength();
    initialDate = initialState.getTimestamp();
    // Align the date to an interval boundary.
    alignedDate = alignDateToIntervalBoundary(initialDate, intervalLength);
    // the previous interval.
    if (alignedDate.compareTo(initialDate) < 0) {
        alignedDate = new Date(alignedDate.getTime() + intervalLength);
    }
    // Create an initial replication state object.
    currentDataState = new ReplicationState(alignedDate, 0);
    // Write out the initial "0" state file.
    replicationStore.saveState(currentDataState);
}