Search in sources :

Example 61 with CommitRequest

use of com.swiftmq.impl.routing.single.smqpr.v400.CommitRequest in project pgadapter by GoogleCloudPlatform.

the class JdbcSimpleModeMockServerTest method testErrorHandlingInExplicitTransactionWithCommit.

@Test
public void testErrorHandlingInExplicitTransactionWithCommit() throws SQLException {
    String sql = String.format("%s; %s; %s; %s; %s;", INSERT_STATEMENT, "BEGIN", UPDATE_STATEMENT, INVALID_DML, "COMMIT");
    try (Connection connection = DriverManager.getConnection(createUrl())) {
        try (java.sql.Statement statement = connection.createStatement()) {
            SQLException exception = assertThrows(SQLException.class, () -> statement.execute(sql));
            assertThat(exception.getMessage(), containsString("INVALID_ARGUMENT: Statement is invalid."));
            // Execute a client side statement to verify that the transaction is in the aborted state.
            exception = assertThrows(SQLException.class, () -> statement.execute("show transaction isolation level"));
            assertTrue(exception.getMessage(), exception.getMessage().contains(TRANSACTION_ABORTED_ERROR));
            // Rollback the transaction and verify that we can get out of the aborted state.
            statement.execute("rollback work");
            assertTrue(statement.execute("show transaction isolation level"));
        }
    }
    // The first DML statement is executed separately, as it is followed by a non-DML statement.
    // The BEGIN statement will switch the batch to use an explicit transaction. The first DML
    // statement will be included as part of that explicit transaction.
    List<ExecuteSqlRequest> requests = mockSpanner.getRequestsOfType(ExecuteSqlRequest.class);
    assertEquals(1, requests.size());
    assertEquals(INSERT_STATEMENT.getSql(), requests.get(0).getSql());
    // Verify that the DML statements were batched together by PgAdapter.
    List<ExecuteBatchDmlRequest> batchDmlRequests = mockSpanner.getRequestsOfType(ExecuteBatchDmlRequest.class);
    assertEquals(1, requests.size());
    ExecuteBatchDmlRequest request = batchDmlRequests.get(0);
    assertEquals(2, request.getStatementsCount());
    assertEquals(UPDATE_STATEMENT.getSql(), request.getStatements(0).getSql());
    assertEquals(INVALID_DML.getSql(), request.getStatements(1).getSql());
    // The explicit transaction is rolled back by PGAdapter. The
    List<RollbackRequest> rollbackRequests = mockSpanner.getRequestsOfType(RollbackRequest.class);
    assertEquals(1, rollbackRequests.size());
    // The transaction should not be committed as it fails on the invalid DML statement.
    List<CommitRequest> commitRequests = mockSpanner.getRequestsOfType(CommitRequest.class);
    assertEquals(0, commitRequests.size());
}
Also used : CommitRequest(com.google.spanner.v1.CommitRequest) SQLException(java.sql.SQLException) Connection(java.sql.Connection) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) RollbackRequest(com.google.spanner.v1.RollbackRequest) ExecuteSqlRequest(com.google.spanner.v1.ExecuteSqlRequest) Statement(java.sql.Statement) ExecuteBatchDmlRequest(com.google.spanner.v1.ExecuteBatchDmlRequest) Test(org.junit.Test)

Example 62 with CommitRequest

use of com.swiftmq.impl.routing.single.smqpr.v400.CommitRequest in project beam by apache.

the class SpannerAccessor method createAndConnect.

private static SpannerAccessor createAndConnect(SpannerConfig spannerConfig) {
    SpannerOptions.Builder builder = SpannerOptions.newBuilder();
    // Set retryable codes for all API methods
    if (spannerConfig.getRetryableCodes() != null) {
        builder.getSpannerStubSettingsBuilder().applyToAllUnaryMethods(input -> {
            input.setRetryableCodes(spannerConfig.getRetryableCodes());
            return null;
        });
        builder.getSpannerStubSettingsBuilder().executeStreamingSqlSettings().setRetryableCodes(spannerConfig.getRetryableCodes());
    }
    // Set commit retry settings
    UnaryCallSettings.Builder<CommitRequest, CommitResponse> commitSettings = builder.getSpannerStubSettingsBuilder().commitSettings();
    ValueProvider<Duration> commitDeadline = spannerConfig.getCommitDeadline();
    if (spannerConfig.getCommitRetrySettings() != null) {
        commitSettings.setRetrySettings(spannerConfig.getCommitRetrySettings());
    } else if (commitDeadline != null && commitDeadline.get().getMillis() > 0) {
        // Set the GRPC deadline on the Commit API call.
        RetrySettings.Builder commitRetrySettingsBuilder = commitSettings.getRetrySettings().toBuilder();
        commitSettings.setRetrySettings(commitRetrySettingsBuilder.setTotalTimeout(org.threeten.bp.Duration.ofMillis(commitDeadline.get().getMillis())).setMaxRpcTimeout(org.threeten.bp.Duration.ofMillis(commitDeadline.get().getMillis())).setInitialRpcTimeout(org.threeten.bp.Duration.ofMillis(commitDeadline.get().getMillis())).build());
    }
    // Set execute streaming sql retry settings
    ServerStreamingCallSettings.Builder<ExecuteSqlRequest, PartialResultSet> executeStreamingSqlSettings = builder.getSpannerStubSettingsBuilder().executeStreamingSqlSettings();
    if (spannerConfig.getExecuteStreamingSqlRetrySettings() != null) {
        executeStreamingSqlSettings.setRetrySettings(spannerConfig.getExecuteStreamingSqlRetrySettings());
    } else {
        // Setting the timeout for streaming read to 2 hours. This is 1 hour by default
        // after BEAM 2.20.
        RetrySettings.Builder executeSqlStreamingRetrySettings = executeStreamingSqlSettings.getRetrySettings().toBuilder();
        executeStreamingSqlSettings.setRetrySettings(executeSqlStreamingRetrySettings.setInitialRpcTimeout(org.threeten.bp.Duration.ofMinutes(120)).setMaxRpcTimeout(org.threeten.bp.Duration.ofMinutes(120)).setTotalTimeout(org.threeten.bp.Duration.ofMinutes(120)).build());
    }
    ValueProvider<String> projectId = spannerConfig.getProjectId();
    if (projectId != null) {
        builder.setProjectId(projectId.get());
    }
    ServiceFactory<Spanner, SpannerOptions> serviceFactory = spannerConfig.getServiceFactory();
    if (serviceFactory != null) {
        builder.setServiceFactory(serviceFactory);
    }
    ValueProvider<String> host = spannerConfig.getHost();
    if (host != null) {
        builder.setHost(host.get());
    }
    ValueProvider<String> emulatorHost = spannerConfig.getEmulatorHost();
    if (emulatorHost != null) {
        builder.setEmulatorHost(emulatorHost.get());
        if (spannerConfig.getIsLocalChannelProvider() != null && spannerConfig.getIsLocalChannelProvider().get()) {
            builder.setChannelProvider(LocalChannelProvider.create(emulatorHost.get()));
        }
        builder.setCredentials(NoCredentials.getInstance());
    }
    String userAgentString = USER_AGENT_PREFIX + "/" + ReleaseInfo.getReleaseInfo().getVersion();
    builder.setHeaderProvider(FixedHeaderProvider.create("user-agent", userAgentString));
    SpannerOptions options = builder.build();
    Spanner spanner = options.getService();
    String instanceId = spannerConfig.getInstanceId().get();
    String databaseId = spannerConfig.getDatabaseId().get();
    DatabaseClient databaseClient = spanner.getDatabaseClient(DatabaseId.of(options.getProjectId(), instanceId, databaseId));
    BatchClient batchClient = spanner.getBatchClient(DatabaseId.of(options.getProjectId(), instanceId, databaseId));
    DatabaseAdminClient databaseAdminClient = spanner.getDatabaseAdminClient();
    return new SpannerAccessor(spanner, databaseClient, databaseAdminClient, batchClient, spannerConfig);
}
Also used : CommitRequest(com.google.spanner.v1.CommitRequest) BatchClient(com.google.cloud.spanner.BatchClient) CommitResponse(com.google.spanner.v1.CommitResponse) Duration(org.joda.time.Duration) SpannerOptions(com.google.cloud.spanner.SpannerOptions) ServerStreamingCallSettings(com.google.api.gax.rpc.ServerStreamingCallSettings) RetrySettings(com.google.api.gax.retrying.RetrySettings) ExecuteSqlRequest(com.google.spanner.v1.ExecuteSqlRequest) DatabaseClient(com.google.cloud.spanner.DatabaseClient) UnaryCallSettings(com.google.api.gax.rpc.UnaryCallSettings) DatabaseAdminClient(com.google.cloud.spanner.DatabaseAdminClient) PartialResultSet(com.google.spanner.v1.PartialResultSet) Spanner(com.google.cloud.spanner.Spanner)

Example 63 with CommitRequest

use of com.swiftmq.impl.routing.single.smqpr.v400.CommitRequest in project beam by apache.

the class DatastoreV1Test method testDatatoreWriterFnWithLargeEntities.

/**
 * Tests {@link DatastoreWriterFn} with large entities that need to be split into more batches.
 */
@Test
public void testDatatoreWriterFnWithLargeEntities() throws Exception {
    List<Mutation> mutations = new ArrayList<>();
    int entitySize = 0;
    for (int i = 0; i < 12; ++i) {
        Entity entity = Entity.newBuilder().setKey(makeKey("key" + i, i + 1)).putProperties("long", makeValue(new String(new char[900_000])).setExcludeFromIndexes(true).build()).build();
        // Take the size of any one entity.
        entitySize = entity.getSerializedSize();
        mutations.add(makeUpsert(entity).build());
    }
    DatastoreWriterFn datastoreWriter = new DatastoreWriterFn(StaticValueProvider.of(PROJECT_ID), null, mockDatastoreFactory, new FakeWriteBatcher());
    DoFnTester<Mutation, Void> doFnTester = DoFnTester.of(datastoreWriter);
    doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
    doFnTester.processBundle(mutations);
    // This test is over-specific currently; it requires that we split the 12 entity writes into 3
    // requests, but we only need each CommitRequest to be less than 10MB in size.
    int entitiesPerRpc = DATASTORE_BATCH_UPDATE_BYTES_LIMIT / entitySize;
    int start = 0;
    while (start < mutations.size()) {
        int end = Math.min(mutations.size(), start + entitiesPerRpc);
        CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
        commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
        commitRequest.addAllMutations(mutations.subList(start, end));
        // Verify all the batch requests were made with the expected mutations.
        verify(mockDatastore).commit(commitRequest.build());
        start = end;
    }
}
Also used : CommitRequest(com.google.datastore.v1.CommitRequest) Entity(com.google.datastore.v1.Entity) DeleteEntity(org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.DeleteEntity) ArrayList(java.util.ArrayList) Mutation(com.google.datastore.v1.Mutation) DatastoreWriterFn(org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.DatastoreWriterFn) Test(org.junit.Test)

Example 64 with CommitRequest

use of com.swiftmq.impl.routing.single.smqpr.v400.CommitRequest in project java-spanner by googleapis.

the class DatabaseClientImplTest method testCommitWithTag.

@Test
public void testCommitWithTag() {
    DatabaseClient client = spanner.getDatabaseClient(DatabaseId.of(TEST_PROJECT, TEST_INSTANCE, TEST_DATABASE));
    TransactionRunner runner = client.readWriteTransaction(Options.tag("app=spanner,env=test,action=commit"));
    runner.run(transaction -> {
        transaction.buffer(Mutation.delete("TEST", KeySet.all()));
        return null;
    });
    List<CommitRequest> requests = mockSpanner.getRequestsOfType(CommitRequest.class);
    assertThat(requests).hasSize(1);
    CommitRequest request = requests.get(0);
    assertNotNull(request.getRequestOptions());
    assertThat(request.getRequestOptions().getRequestTag()).isEmpty();
    assertThat(request.getRequestOptions().getTransactionTag()).isEqualTo("app=spanner,env=test,action=commit");
}
Also used : CommitRequest(com.google.spanner.v1.CommitRequest) Test(org.junit.Test)

Example 65 with CommitRequest

use of com.swiftmq.impl.routing.single.smqpr.v400.CommitRequest in project java-spanner by googleapis.

the class DatabaseClientImplTest method testAsyncRunnerCommitWithTag.

@Test
public void testAsyncRunnerCommitWithTag() {
    DatabaseClient client = spanner.getDatabaseClient(DatabaseId.of(TEST_PROJECT, TEST_INSTANCE, TEST_DATABASE));
    AsyncRunner runner = client.runAsync(Options.tag("app=spanner,env=test,action=runner"));
    get(runner.runAsync(txn -> {
        txn.buffer(Mutation.delete("TEST", KeySet.all()));
        return ApiFutures.immediateFuture(null);
    }, executor));
    List<CommitRequest> requests = mockSpanner.getRequestsOfType(CommitRequest.class);
    assertThat(requests).hasSize(1);
    CommitRequest request = requests.get(0);
    assertNotNull(request.getRequestOptions());
    assertThat(request.getRequestOptions().getRequestTag()).isEmpty();
    assertThat(request.getRequestOptions().getTransactionTag()).isEqualTo("app=spanner,env=test,action=runner");
}
Also used : LocalChannelProvider(com.google.api.gax.grpc.testing.LocalChannelProvider) Context(io.grpc.Context) QueryAnalyzeMode(com.google.cloud.spanner.ReadContext.QueryAnalyzeMode) NoCredentials(com.google.cloud.NoCredentials) Timestamp(com.google.cloud.Timestamp) SettableFuture(com.google.common.util.concurrent.SettableFuture) StatementResult(com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult) ReadRequest(com.google.spanner.v1.ReadRequest) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) After(org.junit.After) Status(io.grpc.Status) READ_ONE_KEY_VALUE_RESULTSET(com.google.cloud.spanner.MockSpannerTestUtil.READ_ONE_KEY_VALUE_RESULTSET) AfterClass(org.junit.AfterClass) ApiFutures(com.google.api.core.ApiFutures) SpannerApiFutures.get(com.google.cloud.spanner.SpannerApiFutures.get) READ_COLUMN_NAMES(com.google.cloud.spanner.MockSpannerTestUtil.READ_COLUMN_NAMES) DeleteSessionRequest(com.google.spanner.v1.DeleteSessionRequest) Set(java.util.Set) ScheduledThreadPoolExecutor(java.util.concurrent.ScheduledThreadPoolExecutor) Logger(java.util.logging.Logger) Executors(java.util.concurrent.Executors) ApiFuture(com.google.api.core.ApiFuture) Duration(org.threeten.bp.Duration) List(java.util.List) ExecuteSqlRequest(com.google.spanner.v1.ExecuteSqlRequest) ResourceNotFoundException(com.google.cloud.spanner.SpannerException.ResourceNotFoundException) QueryMode(com.google.spanner.v1.ExecuteSqlRequest.QueryMode) Mockito.mock(org.mockito.Mockito.mock) Priority(com.google.spanner.v1.RequestOptions.Priority) TransactionOption(com.google.cloud.spanner.Options.TransactionOption) BeforeClass(org.junit.BeforeClass) READ_TABLE_NAME(com.google.cloud.spanner.MockSpannerTestUtil.READ_TABLE_NAME) Stopwatch(com.google.common.base.Stopwatch) Assert.assertThrows(org.junit.Assert.assertThrows) RunWith(org.junit.runner.RunWith) Function(java.util.function.Function) InProcessServerBuilder(io.grpc.inprocess.InProcessServerBuilder) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) RetrySettings(com.google.api.gax.retrying.RetrySettings) CommitRequest(com.google.spanner.v1.CommitRequest) PooledSessionFuture(com.google.cloud.spanner.SessionPool.PooledSessionFuture) ImmutableList(com.google.common.collect.ImmutableList) GrpcStreamIterator(com.google.cloud.spanner.AbstractResultSet.GrpcStreamIterator) Server(io.grpc.Server) SimulatedExecutionTime(com.google.cloud.spanner.MockSpannerServiceImpl.SimulatedExecutionTime) ExecutorService(java.util.concurrent.ExecutorService) READ_ONE_KEY_VALUE_STATEMENT(com.google.cloud.spanner.MockSpannerTestUtil.READ_ONE_KEY_VALUE_STATEMENT) Before(org.junit.Before) QueryOptions(com.google.spanner.v1.ExecuteSqlRequest.QueryOptions) TransactionContextFuture(com.google.cloud.spanner.AsyncTransactionManager.TransactionContextFuture) CallbackResponse(com.google.cloud.spanner.AsyncResultSet.CallbackResponse) AbstractMessage(com.google.protobuf.AbstractMessage) SpannerCallContextTimeoutConfigurator(com.google.cloud.spanner.SpannerOptions.SpannerCallContextTimeoutConfigurator) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) JUnit4(org.junit.runners.JUnit4) Truth.assertThat(com.google.common.truth.Truth.assertThat) Mockito.verify(org.mockito.Mockito.verify) StatusRuntimeException(io.grpc.StatusRuntimeException) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Assert.assertNull(org.junit.Assert.assertNull) RpcPriority(com.google.cloud.spanner.Options.RpcPriority) SELECT1(com.google.cloud.spanner.MockSpannerTestUtil.SELECT1) ExecuteBatchDmlRequest(com.google.spanner.v1.ExecuteBatchDmlRequest) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) CommitRequest(com.google.spanner.v1.CommitRequest) Test(org.junit.Test)

Aggregations

Test (org.junit.Test)85 CommitRequest (com.google.spanner.v1.CommitRequest)47 CommitRequest (com.google.firestore.v1.CommitRequest)40 CommitResponse (com.google.firestore.v1.CommitResponse)40 ArrayList (java.util.ArrayList)26 Connection (java.sql.Connection)23 ByteString (com.google.protobuf.ByteString)14 ExecuteSqlRequest (com.google.spanner.v1.ExecuteSqlRequest)12 SQLException (java.sql.SQLException)12 Statement (java.sql.Statement)12 CopyManager (org.postgresql.copy.CopyManager)11 BaseConnection (org.postgresql.core.BaseConnection)11 ExecuteBatchDmlRequest (com.google.spanner.v1.ExecuteBatchDmlRequest)10 ArrayValue (com.google.firestore.v1.ArrayValue)9 MapValue (com.google.firestore.v1.MapValue)9 Value (com.google.firestore.v1.Value)9 Write (com.google.firestore.v1.Write)9 StringReader (java.io.StringReader)9 HashMap (java.util.HashMap)9 AbstractMessage (com.google.protobuf.AbstractMessage)8