Search in sources :

Example 1 with Response

use of org.neo4j.com.Response in project neo4j by neo4j.

the class SwitchToSlaveCopyThenBranchTest method newSwitchToSlaveSpy.

@SuppressWarnings("unchecked")
private SwitchToSlaveCopyThenBranch newSwitchToSlaveSpy(PageCache pageCacheMock, StoreCopyClient storeCopyClient) throws IOException {
    ClusterMembers clusterMembers = mock(ClusterMembers.class);
    ClusterMember master = mock(ClusterMember.class);
    when(master.getStoreId()).thenReturn(storeId);
    when(master.getHARole()).thenReturn(HighAvailabilityModeSwitcher.MASTER);
    when(master.hasRole(eq(HighAvailabilityModeSwitcher.MASTER))).thenReturn(true);
    when(master.getInstanceId()).thenReturn(new InstanceId(1));
    when(clusterMembers.getMembers()).thenReturn(asList(master));
    Dependencies resolver = new Dependencies();
    resolver.satisfyDependencies(requestContextFactory, clusterMembers, mock(TransactionObligationFulfiller.class), mock(OnlineBackupKernelExtension.class), mock(IndexConfigStore.class), mock(TransactionCommittingResponseUnpacker.class), mock(DataSourceManager.class), mock(StoreLockerLifecycleAdapter.class), mock(FileSystemWatcherService.class));
    NeoStoreDataSource dataSource = mock(NeoStoreDataSource.class);
    when(dataSource.getStoreId()).thenReturn(storeId);
    TransactionStats transactionCounters = mock(TransactionStats.class);
    when(transactionCounters.getNumberOfActiveTransactions()).thenReturn(0L);
    Response<HandshakeResult> response = mock(Response.class);
    when(response.response()).thenReturn(new HandshakeResult(42, 2));
    when(masterClient.handshake(anyLong(), any(StoreId.class))).thenReturn(response);
    when(masterClient.getProtocolVersion()).thenReturn(MasterClient.CURRENT);
    TransactionIdStore transactionIdStoreMock = mock(TransactionIdStore.class);
    // note that the checksum (the second member of the array) is the same as the one in the handshake mock above
    when(transactionIdStoreMock.getLastCommittedTransaction()).thenReturn(new TransactionId(42, 42, 42));
    MasterClientResolver masterClientResolver = mock(MasterClientResolver.class);
    when(masterClientResolver.instantiate(anyString(), anyInt(), anyString(), any(Monitors.class), any(StoreId.class), any(LifeSupport.class))).thenReturn(masterClient);
    return spy(new SwitchToSlaveCopyThenBranch(new File(""), NullLogService.getInstance(), configMock(), resolver, mock(HaIdGeneratorFactory.class), mock(DelegateInvocationHandler.class), mock(ClusterMemberAvailability.class), requestContextFactory, pullerFactory, masterClientResolver, mock(SwitchToSlave.Monitor.class), storeCopyClient, Suppliers.singleton(dataSource), Suppliers.singleton(transactionIdStoreMock), slave -> {
        SlaveServer server = mock(SlaveServer.class);
        InetSocketAddress inetSocketAddress = InetSocketAddress.createUnresolved("localhost", 42);
        when(server.getSocketAddress()).thenReturn(inetSocketAddress);
        return server;
    }, updatePuller, pageCacheMock, mock(Monitors.class), transactionCounters));
}
Also used : InstanceId(org.neo4j.cluster.InstanceId) StoreId(org.neo4j.kernel.impl.store.StoreId) NeoStoreDataSource(org.neo4j.kernel.NeoStoreDataSource) MasterClientResolver(org.neo4j.kernel.ha.com.slave.MasterClientResolver) Suppliers(org.neo4j.function.Suppliers) URISyntaxException(java.net.URISyntaxException) OnlineBackupKernelExtension(org.neo4j.backup.OnlineBackupKernelExtension) DelegateInvocationHandler(org.neo4j.kernel.ha.DelegateInvocationHandler) Dependencies(org.neo4j.kernel.impl.util.Dependencies) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) NullLogProvider(org.neo4j.logging.NullLogProvider) MasterClient(org.neo4j.kernel.ha.com.slave.MasterClient) StoreIdTestFactory.newStoreIdForCurrentVersion(org.neo4j.com.StoreIdTestFactory.newStoreIdForCurrentVersion) HighAvailabilityModeSwitcher(org.neo4j.kernel.ha.cluster.modeswitch.HighAvailabilityModeSwitcher) Mockito.doThrow(org.mockito.Mockito.doThrow) TransactionObligationFulfiller(org.neo4j.com.storecopy.TransactionObligationFulfiller) Arrays.asList(java.util.Arrays.asList) Matchers.eq(org.mockito.Matchers.eq) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Matchers.anyInt(org.mockito.Matchers.anyInt) Assert.fail(org.junit.Assert.fail) URI(java.net.URI) MismatchingStoreIdException(org.neo4j.kernel.impl.store.MismatchingStoreIdException) Response(org.neo4j.com.Response) PageCache(org.neo4j.io.pagecache.PageCache) Predicate(java.util.function.Predicate) InetSocketAddress(java.net.InetSocketAddress) TransactionCommittingResponseUnpacker(org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker) UpdatePuller(org.neo4j.kernel.ha.UpdatePuller) UpdatePullerScheduler(org.neo4j.kernel.ha.UpdatePullerScheduler) Matchers.any(org.mockito.Matchers.any) Stream(java.util.stream.Stream) ClusterMemberAvailability(org.neo4j.cluster.member.ClusterMemberAvailability) Mockito.withSettings(org.mockito.Mockito.withSettings) RequestContextFactory(org.neo4j.kernel.ha.com.RequestContextFactory) SlaveUpdatePuller(org.neo4j.kernel.ha.SlaveUpdatePuller) Mockito.mock(org.mockito.Mockito.mock) BranchedDataException(org.neo4j.kernel.ha.BranchedDataException) BranchedDataPolicy(org.neo4j.kernel.ha.BranchedDataPolicy) MoveAfterCopy(org.neo4j.com.storecopy.MoveAfterCopy) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) PagedFile(org.neo4j.io.pagecache.PagedFile) Monitors(org.neo4j.kernel.monitoring.Monitors) HandshakeResult(org.neo4j.kernel.ha.com.master.HandshakeResult) JobScheduler(org.neo4j.kernel.impl.util.JobScheduler) Mockito.spy(org.mockito.Mockito.spy) PullerFactory(org.neo4j.kernel.ha.PullerFactory) Matchers.anyString(org.mockito.Matchers.anyString) TransactionId(org.neo4j.kernel.impl.store.TransactionId) ClusterMembers(org.neo4j.kernel.ha.cluster.member.ClusterMembers) Answer(org.mockito.stubbing.Answer) InvocationOnMock(org.mockito.invocation.InvocationOnMock) StoreLockerLifecycleAdapter(org.neo4j.kernel.internal.StoreLockerLifecycleAdapter) SlaveServer(org.neo4j.kernel.ha.com.slave.SlaveServer) HaIdGeneratorFactory(org.neo4j.kernel.ha.id.HaIdGeneratorFactory) CancellationRequest(org.neo4j.helpers.CancellationRequest) Matchers.anyLong(org.mockito.Matchers.anyLong) MapUtil.stringMap(org.neo4j.helpers.collection.MapUtil.stringMap) Lifecycle(org.neo4j.kernel.lifecycle.Lifecycle) Config(org.neo4j.kernel.configuration.Config) InOrder(org.mockito.InOrder) Test(org.junit.Test) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) DataSourceManager(org.neo4j.kernel.impl.transaction.state.DataSourceManager) File(java.io.File) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Mockito(org.mockito.Mockito) NullLogService(org.neo4j.kernel.impl.logging.NullLogService) Mockito.never(org.mockito.Mockito.never) IndexConfigStore(org.neo4j.kernel.impl.index.IndexConfigStore) Assert.assertNull(org.junit.Assert.assertNull) ClusterSettings(org.neo4j.cluster.ClusterSettings) StoreCopyClient(org.neo4j.com.storecopy.StoreCopyClient) TransactionStats(org.neo4j.kernel.impl.transaction.TransactionStats) ClusterMember(org.neo4j.kernel.ha.cluster.member.ClusterMember) FileSystemWatcherService(org.neo4j.kernel.impl.util.watcher.FileSystemWatcherService) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) HandshakeResult(org.neo4j.kernel.ha.com.master.HandshakeResult) TransactionObligationFulfiller(org.neo4j.com.storecopy.TransactionObligationFulfiller) InetSocketAddress(java.net.InetSocketAddress) OnlineBackupKernelExtension(org.neo4j.backup.OnlineBackupKernelExtension) ClusterMembers(org.neo4j.kernel.ha.cluster.member.ClusterMembers) DataSourceManager(org.neo4j.kernel.impl.transaction.state.DataSourceManager) SlaveServer(org.neo4j.kernel.ha.com.slave.SlaveServer) ClusterMember(org.neo4j.kernel.ha.cluster.member.ClusterMember) StoreId(org.neo4j.kernel.impl.store.StoreId) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) Dependencies(org.neo4j.kernel.impl.util.Dependencies) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) InstanceId(org.neo4j.cluster.InstanceId) NeoStoreDataSource(org.neo4j.kernel.NeoStoreDataSource) IndexConfigStore(org.neo4j.kernel.impl.index.IndexConfigStore) TransactionCommittingResponseUnpacker(org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker) TransactionId(org.neo4j.kernel.impl.store.TransactionId) MasterClientResolver(org.neo4j.kernel.ha.com.slave.MasterClientResolver) TransactionStats(org.neo4j.kernel.impl.transaction.TransactionStats) FileSystemWatcherService(org.neo4j.kernel.impl.util.watcher.FileSystemWatcherService) Monitors(org.neo4j.kernel.monitoring.Monitors) StoreLockerLifecycleAdapter(org.neo4j.kernel.internal.StoreLockerLifecycleAdapter) PagedFile(org.neo4j.io.pagecache.PagedFile) File(java.io.File)

Example 2 with Response

use of org.neo4j.com.Response in project neo4j by neo4j.

the class SlaveUpdatePullerTest method shouldCapExcessiveInvalidEpochExceptionLogging.

@Test
public void shouldCapExcessiveInvalidEpochExceptionLogging() throws Exception {
    OngoingStubbing<Response<Void>> updatePullStubbing = when(master.pullUpdates(any(RequestContext.class)));
    updatePullStubbing.thenThrow(new InvalidEpochException(2, 1));
    for (int i = 0; i < SlaveUpdatePuller.LOG_CAP + 20; i++) {
        updatePuller.pullUpdates();
    }
    logProvider.assertContainsThrowablesMatching(0, repeat(new InvalidEpochException(2, 1), SlaveUpdatePuller.LOG_CAP));
    // And we should be able to recover afterwards
    updatePullStubbing.thenReturn(Response.EMPTY).thenThrow(new InvalidEpochException(2, 1));
    // This one will succeed and unlock the circuit breaker
    updatePuller.pullUpdates();
    // And then we log another exception
    updatePuller.pullUpdates();
    logProvider.assertContainsThrowablesMatching(0, repeat(new InvalidEpochException(2, 1), SlaveUpdatePuller.LOG_CAP + 1));
}
Also used : Response(org.neo4j.com.Response) InvalidEpochException(org.neo4j.kernel.ha.com.master.InvalidEpochException) Test(org.junit.Test)

Example 3 with Response

use of org.neo4j.com.Response in project neo4j by neo4j.

the class StoreCopyResponsePacker method packTransactionStreamResponse.

@Override
public <T> Response<T> packTransactionStreamResponse(RequestContext context, T response) {
    final long toStartFrom = mandatoryStartTransactionId;
    final long toEndAt = transactionIdStore.getLastCommittedTransactionId();
    TransactionStream transactions = visitor -> {
        if (toStartFrom > BASE_TX_ID && toStartFrom <= toEndAt) {
            monitor.startStreamingTransactions(toStartFrom);
            extractTransactions(toStartFrom, filterVisitor(visitor, toEndAt));
            monitor.finishStreamingTransactions(toEndAt);
        }
    };
    return new TransactionStreamResponse<>(response, storeId.get(), transactions, ResourceReleaser.NO_OP);
}
Also used : Response(org.neo4j.com.Response) StoreId(org.neo4j.kernel.impl.store.StoreId) TransactionStream(org.neo4j.com.TransactionStream) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) NoSuchTransactionException(org.neo4j.kernel.impl.transaction.log.NoSuchTransactionException) Supplier(java.util.function.Supplier) RequestContext(org.neo4j.com.RequestContext) TransactionStreamResponse(org.neo4j.com.TransactionStreamResponse) ResponsePacker(org.neo4j.com.storecopy.ResponsePacker) CommittedTransactionRepresentation(org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation) LogFileInformation(org.neo4j.kernel.impl.transaction.log.LogFileInformation) Monitor(org.neo4j.com.storecopy.StoreCopyServer.Monitor) StoreCopyServer(org.neo4j.com.storecopy.StoreCopyServer) ResourceReleaser(org.neo4j.com.ResourceReleaser) Visitor(org.neo4j.helpers.collection.Visitor) BASE_TX_ID(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_ID) TransactionStreamResponse(org.neo4j.com.TransactionStreamResponse) TransactionStream(org.neo4j.com.TransactionStream)

Example 4 with Response

use of org.neo4j.com.Response in project neo4j by neo4j.

the class StoreCopyClient method writeTransactionsToActiveLogFile.

private void writeTransactionsToActiveLogFile(File tempStoreDir, Response<?> response) throws Exception {
    LifeSupport life = new LifeSupport();
    try {
        // Start the log and appender
        PhysicalLogFiles logFiles = new PhysicalLogFiles(tempStoreDir, fs);
        LogHeaderCache logHeaderCache = new LogHeaderCache(10);
        ReadOnlyLogVersionRepository logVersionRepository = new ReadOnlyLogVersionRepository(pageCache, tempStoreDir);
        ReadOnlyTransactionIdStore readOnlyTransactionIdStore = new ReadOnlyTransactionIdStore(pageCache, tempStoreDir);
        LogFile logFile = life.add(new PhysicalLogFile(fs, logFiles, Long.MAX_VALUE, /*don't rotate*/
        readOnlyTransactionIdStore::getLastCommittedTransactionId, logVersionRepository, new Monitors().newMonitor(PhysicalLogFile.Monitor.class), logHeaderCache));
        life.start();
        // Just write all transactions to the active log version. Remember that this is after a store copy
        // where there are no logs, and the transaction stream we're about to write will probably contain
        // transactions that goes some time back, before the last committed transaction id. So we cannot
        // use a TransactionAppender, since it has checks for which transactions one can append.
        FlushableChannel channel = logFile.getWriter();
        final TransactionLogWriter writer = new TransactionLogWriter(new LogEntryWriter(channel));
        final AtomicLong firstTxId = new AtomicLong(BASE_TX_ID);
        response.accept(new Response.Handler() {

            @Override
            public void obligation(long txId) throws IOException {
                throw new UnsupportedOperationException("Shouldn't be called");
            }

            @Override
            public Visitor<CommittedTransactionRepresentation, Exception> transactions() {
                return transaction -> {
                    long txId = transaction.getCommitEntry().getTxId();
                    if (firstTxId.compareAndSet(BASE_TX_ID, txId)) {
                        monitor.startReceivingTransactions(txId);
                    }
                    writer.append(transaction.getTransactionRepresentation(), txId);
                    return false;
                };
            }
        });
        long endTxId = firstTxId.get();
        if (endTxId != BASE_TX_ID) {
            monitor.finishReceivingTransactions(endTxId);
        }
        long currentLogVersion = logVersionRepository.getCurrentLogVersion();
        writer.checkPoint(new LogPosition(currentLogVersion, LOG_HEADER_SIZE));
        // And since we write this manually we need to set the correct transaction id in the
        // header of the log that we just wrote.
        File currentLogFile = logFiles.getLogFileForVersion(currentLogVersion);
        writeLogHeader(fs, currentLogFile, currentLogVersion, max(BASE_TX_ID, endTxId - 1));
        if (!forensics) {
            // since we just create new log and put checkpoint into it with offset equals to
            // LOG_HEADER_SIZE we need to update last transaction offset to be equal to this newly defined max
            // offset otherwise next checkpoint that use last transaction offset will be created for non
            // existing offset that is in most of the cases bigger than new log size.
            // Recovery will treat that as last checkpoint and will not try to recover store till new
            // last closed transaction offset will not overcome old one. Till that happens it will be
            // impossible for recovery process to restore the store
            File neoStore = new File(tempStoreDir, MetaDataStore.DEFAULT_NAME);
            MetaDataStore.setRecord(pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_BYTE_OFFSET, LOG_HEADER_SIZE);
        }
    } finally {
        life.shutdown();
    }
}
Also used : Visitor(org.neo4j.helpers.collection.Visitor) IOException(java.io.IOException) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) LogFile(org.neo4j.kernel.impl.transaction.log.LogFile) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) Response(org.neo4j.com.Response) AtomicLong(java.util.concurrent.atomic.AtomicLong) ReadOnlyTransactionIdStore(org.neo4j.kernel.impl.transaction.log.ReadOnlyTransactionIdStore) FlushableChannel(org.neo4j.kernel.impl.transaction.log.FlushableChannel) Monitors(org.neo4j.kernel.monitoring.Monitors) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) TransactionLogWriter(org.neo4j.kernel.impl.transaction.log.TransactionLogWriter) LogEntryWriter(org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter) ReadOnlyLogVersionRepository(org.neo4j.kernel.impl.transaction.log.ReadOnlyLogVersionRepository) LogHeaderCache(org.neo4j.kernel.impl.transaction.log.LogHeaderCache) LogFile(org.neo4j.kernel.impl.transaction.log.LogFile) File(java.io.File) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition)

Example 5 with Response

use of org.neo4j.com.Response in project neo4j by neo4j.

the class MasterClient214 method allocateIds.

@Override
public Response<IdAllocation> allocateIds(RequestContext context, final IdType idType) {
    Serializer serializer = buffer -> buffer.writeByte(idType.ordinal());
    Deserializer<IdAllocation> deserializer = (buffer, temporaryBuffer) -> readIdAllocation(buffer);
    return sendRequest(requestTypes.type(HaRequestTypes.Type.ALLOCATE_IDS), context, serializer, deserializer);
}
Also used : VOID_DESERIALIZER(org.neo4j.com.Protocol.VOID_DESERIALIZER) StoreId(org.neo4j.kernel.impl.store.StoreId) IdAllocation(org.neo4j.kernel.ha.id.IdAllocation) RequestType(org.neo4j.com.RequestType) LockResult(org.neo4j.kernel.ha.lock.LockResult) ObjectSerializer(org.neo4j.com.ObjectSerializer) ChannelBuffer(org.jboss.netty.buffer.ChannelBuffer) ResourceType(org.neo4j.storageengine.api.lock.ResourceType) Exceptions(org.neo4j.helpers.Exceptions) StoreWriter(org.neo4j.com.storecopy.StoreWriter) LogProvider(org.neo4j.logging.LogProvider) Serializer(org.neo4j.com.Serializer) HandshakeResult(org.neo4j.kernel.ha.com.master.HandshakeResult) IdRange(org.neo4j.kernel.impl.store.id.IdRange) RequestContext(org.neo4j.com.RequestContext) RequestMonitor(org.neo4j.com.monitor.RequestMonitor) ReadableClosablePositionAwareChannel(org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel) INTERNAL_PROTOCOL_VERSION(org.neo4j.com.ProtocolVersion.INTERNAL_PROTOCOL_VERSION) MasterClient(org.neo4j.kernel.ha.com.slave.MasterClient) ByteCounterMonitor(org.neo4j.kernel.monitoring.ByteCounterMonitor) Client(org.neo4j.com.Client) EMPTY_SERIALIZER(org.neo4j.com.Protocol.EMPTY_SERIALIZER) Response(org.neo4j.com.Response) Protocol(org.neo4j.com.Protocol) TransactionRepresentation(org.neo4j.kernel.impl.transaction.TransactionRepresentation) LogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader) IOException(java.io.IOException) Protocol214(org.neo4j.com.Protocol214) MasterServer(org.neo4j.kernel.ha.com.master.MasterServer) LockStatus(org.neo4j.kernel.ha.lock.LockStatus) String.format(java.lang.String.format) IdType(org.neo4j.kernel.impl.store.id.IdType) ResponseUnpacker(org.neo4j.com.storecopy.ResponseUnpacker) Protocol.writeString(org.neo4j.com.Protocol.writeString) ProtocolVersion(org.neo4j.com.ProtocolVersion) Protocol.readString(org.neo4j.com.Protocol.readString) Deserializer(org.neo4j.com.Deserializer) Master(org.neo4j.kernel.ha.com.master.Master) IdAllocation(org.neo4j.kernel.ha.id.IdAllocation) ObjectSerializer(org.neo4j.com.ObjectSerializer) Serializer(org.neo4j.com.Serializer)

Aggregations

Response (org.neo4j.com.Response)17 IOException (java.io.IOException)12 StoreId (org.neo4j.kernel.impl.store.StoreId)12 RequestContext (org.neo4j.com.RequestContext)11 HandshakeResult (org.neo4j.kernel.ha.com.master.HandshakeResult)10 MasterClient (org.neo4j.kernel.ha.com.slave.MasterClient)10 Test (org.junit.Test)7 ResponseUnpacker (org.neo4j.com.storecopy.ResponseUnpacker)7 StoreWriter (org.neo4j.com.storecopy.StoreWriter)7 Master (org.neo4j.kernel.ha.com.master.Master)7 String.format (java.lang.String.format)6 ChannelBuffer (org.jboss.netty.buffer.ChannelBuffer)6 Client (org.neo4j.com.Client)6 Deserializer (org.neo4j.com.Deserializer)6 ObjectSerializer (org.neo4j.com.ObjectSerializer)6 Protocol (org.neo4j.com.Protocol)6 EMPTY_SERIALIZER (org.neo4j.com.Protocol.EMPTY_SERIALIZER)6 VOID_DESERIALIZER (org.neo4j.com.Protocol.VOID_DESERIALIZER)6 Protocol.readString (org.neo4j.com.Protocol.readString)6 Protocol.writeString (org.neo4j.com.Protocol.writeString)6