Add Microsoft Entra ID and AWS IAM token authentication to PostgreSQL connections

.github/actions/cloud-database/aurora-postgresql.mts

@@ -1,10 +1,11 @@
 // Provisions and tears down an Aurora PostgreSQL cluster for one run of the cloud database tests.
 
-import { step, run, capture, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, verifyDatabase, teardownStep } from './common.mts';
+import { step, run, capture, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, setTokenConnectionString, runScript, verifyDatabase, teardownStep } from './common.mts';
 import * as aws from './aws.mts';
 
 const databaseName = 'servicecontrol';
 const adminUser = 'sctestadmin';
+const iamUser = 'sc_iam';
 const port = 5432;
 
 function resources(name: string) {
@@ -36,6 +37,7 @@ async function provision(name: string): Promise<void> {
         '--db-subnet-group-name', aws.dbSubnetGroupName(),
         '--no-deletion-protection',
         '--backup-retention-period', '1',
+        '--enable-iam-database-authentication',
         ...tags,
         '--no-cli-pager']);
 
@@ -61,7 +63,11 @@ async function provision(name: string): Promise<void> {
     step('Waiting until the database accepts connections');
     verifyDatabase('PostgreSql', connectionString);
 
+    step(`Creating ${iamUser}, which signs in with an IAM token`);
+    runScript('./create-iam-user-postgresql.cs', [connectionString, iamUser]);
+
     setPersistenceConnectionString('PostgreSql', connectionString);
+    setTokenConnectionString('PostgreSql', `Host=${endpoint};Port=${port};Database=${databaseName};Username=${iamUser};Ssl Mode=Require;Trust Server Certificate=true;Timeout=60;Token Authentication=AwsRdsIam`);
 }
 
 function teardown(name: string): void {

.github/actions/cloud-database/aws-iam-policy.json

@@ -54,6 +54,14 @@
         "rds:AddTagsToResource"
       ],
       "Resource": "*"
+    },
+    {
+      "Sid": "ConnectAsTestIamUser",
+      "Effect": "Allow",
+      "Action": [
+        "rds-db:connect"
+      ],
+      "Resource": "arn:aws:rds-db:*:*:dbuser:*/sc_iam"
     }
   ]
 }

.github/actions/cloud-database/azure-postgresql.mts

@@ -1,10 +1,11 @@
-import { step, run, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, verifyDatabase, teardownStep } from './common.mts';
+import { step, run, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, setTokenConnectionString, verifyDatabase, teardownStep } from './common.mts';
 import * as azure from './azure.mts';
 
 const databaseName = 'servicecontrol';
 const adminUser = 'sctestadmin';
 
 async function provision(name: string): Promise<void> {
+    const principal = azure.signedInPrincipal();
     const password = newAdminPassword();
     const runnerIp = await runnerIpAddress();
     const location = azure.location();
@@ -17,6 +18,8 @@ async function provision(name: string): Promise<void> {
         '--location', location,
         '--admin-user', adminUser,
         `--admin-password=${password}`,
+        '--password-auth', 'Enabled',
+        '--microsoft-entra-auth', 'Enabled',
         '--tier', 'GeneralPurpose',
         '--sku-name', 'Standard_D8ds_v5',
         '--storage-size', '512',
@@ -27,6 +30,16 @@ async function provision(name: string): Promise<void> {
         '--yes',
         '--only-show-errors', '--output', 'none']);
 
+    // The display name becomes the PostgreSQL role, which is the Username a token login gives.
+    step(`Making service principal ${principal.clientId} a Microsoft Entra admin of ${name}`);
+    run('az', ['postgres', 'flexible-server', 'microsoft-entra-admin', 'create',
+        '--resource-group', azure.resourceGroup,
+        '--server-name', name,
+        '--display-name', principal.clientId,
+        '--object-id', principal.objectId,
+        '--type', 'ServicePrincipal',
+        '--only-show-errors', '--output', 'none']);
+
     step(`Creating database ${databaseName}`);
     run('az', ['postgres', 'flexible-server', 'db', 'create',
         '--name', databaseName,
@@ -40,6 +53,7 @@ async function provision(name: string): Promise<void> {
     verifyDatabase('PostgreSql', connectionString);
 
     setPersistenceConnectionString('PostgreSql', connectionString);
+    setTokenConnectionString('PostgreSql', `Host=${name}.postgres.database.azure.com;Port=5432;Database=${databaseName};Username=${principal.clientId};Ssl Mode=Require;Timeout=60;Token Authentication=AzureEntraId`);
 }
 
 function teardown(name: string): void {

.github/actions/cloud-database/azure-sqlserver.mts

@@ -1,22 +1,27 @@
-import { step, run, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, verifyDatabase, teardownStep } from './common.mts';
+import { step, run, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, setTokenConnectionString, verifyDatabase, teardownStep } from './common.mts';
 import * as azure from './azure.mts';
 
 const databaseName = 'servicecontrol';
 const adminUser = 'sctestadmin';
 
 async function provision(name: string): Promise<void> {
+    const principal = azure.signedInPrincipal();
     const password = newAdminPassword();
     const runnerIp = await runnerIpAddress();
     const location = azure.location();
     const tags = azure.tags(name);
 
-    step(`Creating SQL server ${name} in ${azure.resourceGroup} (${location})`);
+    // For an application the sid is its client ID, where PostgreSQL takes the object ID.
+    step(`Creating SQL server ${name} in ${azure.resourceGroup} (${location}), with service principal ${principal.clientId} as its Microsoft Entra admin`);
     run('az', ['sql', 'server', 'create',
         '--name', name,
         '--resource-group', azure.resourceGroup,
         '--location', location,
         '--admin-user', adminUser,
         `--admin-password=${password}`,
+        '--external-admin-principal-type', 'Application',
+        '--external-admin-name', principal.clientId,
+        '--external-admin-sid', principal.clientId,
         ...tags,
         '--only-show-errors', '--output', 'none']);
 
@@ -45,6 +50,7 @@ async function provision(name: string): Promise<void> {
     verifyDatabase('SqlServer', connectionString);
 
     setPersistenceConnectionString('SqlServer', connectionString);
+    setTokenConnectionString('SqlServer', `Server=tcp:${name}.database.windows.net,1433;Initial Catalog=${databaseName};Authentication=Active Directory Default;Encrypt=True;TrustServerCertificate=False;Connect Timeout=60`);
 }
 
 function teardown(name: string): void {

.github/actions/cloud-database/azure.mts

@@ -20,4 +20,24 @@ function location(): string {
     return value;
 }
 
-export { resourceGroup, tags, location };
+// Read from the CLI's own access token, because looking the principal up in Microsoft Graph needs a directory permission that the login is not guaranteed to have.
+function signedInPrincipal(): { objectId: string; clientId: string } {
+    const type = capture('az', ['account', 'show', '--query', 'user.type', '--output', 'tsv']);
+
+    if (type !== 'servicePrincipal') {
+        throw new Error(`The Azure CLI is signed in as a ${type}, and the Microsoft Entra admin has to be a service principal.`);
+    }
+
+    const token = capture('az', ['account', 'get-access-token', '--query', 'accessToken', '--output', 'tsv']);
+    const claims = JSON.parse(Buffer.from(token.split('.')[1] ?? '', 'base64url').toString('utf8'));
+    const objectId: string | undefined = claims.oid;
+    const clientId: string | undefined = claims.appid ?? claims.azp;
+
+    if (!objectId || !clientId) {
+        throw new Error('The Azure CLI access token has no oid or appid claim, so the signed-in service principal cannot be identified.');
+    }
+
+    return { objectId, clientId };
+}
+
+export { resourceGroup, tags, location, signedInPrincipal };

.github/actions/cloud-database/common.mts

@@ -76,17 +76,26 @@ async function runnerIpAddress(): Promise<string> {
     return (await response.text()).trim();
 }
 
-function setPersistenceConnectionString(provider: Provider, connectionString: string): void {
-    const name = `ServiceControl_Persistence_${provider}_ConnectionString`;
-    appendFileSync(process.env.GITHUB_ENV, `${name}=${connectionString}\n`);
+function exportVariable(name: string, value: string): void {
+    appendFileSync(process.env.GITHUB_ENV, `${name}=${value}\n`);
     step(`Exported ${name}`);
 }
 
+function setPersistenceConnectionString(provider: Provider, connectionString: string): void {
+    exportVariable(`ServiceControl_Persistence_${provider}_ConnectionString`, connectionString);
+}
+
+function setTokenConnectionString(provider: Provider, connectionString: string): void {
+    exportVariable(`ServiceControl_Persistence_${provider}_TokenConnectionString`, connectionString);
+}
+
+function runScript(script: string, args: string[]): void {
+    execFileSync('dotnet', ['run', script, '--', ...args], { cwd: import.meta.dirname, stdio: 'inherit' });
+}
+
 // Waits until the database is really reachable.
 function verifyDatabase(provider: Provider, connectionString: string): void {
-    const script = provider === 'SqlServer' ? './verify-sqlserver.cs' : './verify-postgresql.cs';
-
-    execFileSync('dotnet', ['run', script, '--', connectionString], { cwd: import.meta.dirname, stdio: 'inherit' });
+    runScript(provider === 'SqlServer' ? './verify-sqlserver.cs' : './verify-postgresql.cs', [connectionString]);
 }
 
 // Teardown runs even when provisioning failed part way, so it has to tolerate resources that were
@@ -110,6 +119,8 @@ export {
     newAdminPassword,
     runnerIpAddress,
     setPersistenceConnectionString,
+    setTokenConnectionString,
+    runScript,
     verifyDatabase,
     teardownStep
 };

.github/actions/cloud-database/create-iam-user-postgresql.cs

@@ -0,0 +1,36 @@
+#:package [email protected]
+
+// Creates a login that signs in with an IAM token and can create its own schema in the test database. A role that is granted rds_iam can no longer sign in with a password.
+
+using Npgsql;
+
+if (args.Length != 2)
+{
+    Console.Error.WriteLine("usage: dotnet run create-iam-user-postgresql.cs -- <master-connection-string> <user>");
+    return 1;
+}
+
+var builder = new NpgsqlConnectionStringBuilder(args[0]);
+var user = args[1];
+
+if (string.Equals(user, builder.Username, StringComparison.Ordinal))
+{
+    Console.Error.WriteLine($"Refusing to grant rds_iam to '{user}', the user this script signs in as.");
+    return 1;
+}
+
+await using var connection = new NpgsqlConnection(builder.ConnectionString);
+await connection.OpenAsync();
+
+await using var command = connection.CreateCommand();
+command.CommandText = $"""
+    CREATE ROLE {QuoteIdentifier(user)} WITH LOGIN;
+    GRANT rds_iam TO {QuoteIdentifier(user)};
+    GRANT CREATE ON DATABASE {QuoteIdentifier(builder.Database!)} TO {QuoteIdentifier(user)};
+    """;
+await command.ExecuteNonQueryAsync();
+
+Console.WriteLine($"Created '{user}' on {builder.Host}, which signs in to '{builder.Database}' with an IAM token.");
+return 0;
+
+static string QuoteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";

.github/actions/cloud-database/verify-sqlserver.cs

@@ -1,4 +1,4 @@
-#:package Microsoft.Data.SqlClient@6.1.1
+#:package Microsoft.Data.SqlClient@7.1.1
 
 // Waits until the server accepts connections, creates the test database if the provisioning CLI
 // could not, fails the run if the server has no Full-Text Search, and waits until a full-text

docs/README.md

@@ -14,7 +14,7 @@ ServiceControl is the monitoring component of the Particular Service Platform: i
 
 - [ServiceControl documentation](https://docs.particular.net/servicecontrol/) — public documentation entry point
 - [README.md](../README.md) — how to run and debug ServiceControl, ServiceControl.Audit, and ServiceControl.Monitoring locally
-- [Testing overview](testing.md) — index of test kinds, local setup for each, and the scenario checklists to rerun after a change
+- [Testing overview](testing.md) — index of test kinds, local setup for each, the cloud database tests, and the scenario checklists to rerun after a change
 - [Coding and design guidelines](coding-and-design-guidelines.md) — conventions for new code
 - [Deployment](deployment.md) and [Packaging](packaging.md) — how instances are packaged and deployed
 

docs/cloud-database-tests.md

@@ -0,0 +1,26 @@
+# Cloud database tests
+
+The `Cloud database tests` workflow (`.github/workflows/cloud-database-tests.yml`) runs the SQL Server and PostgreSQL persistence and acceptance suites against managed databases. Every target runs nightly at 03:17 UTC on the default branch, and also whenever the release workflow calls it. To run it yourself, start it from the Actions tab and choose one target, or `all`. Starting a run cancels any run of the workflow already in progress on the same branch. The nightly run skips a commit that has already passed a nightly run, and posts to the `pulse-and-control` Slack channel when a target fails or is cancelled. For each target, the `.github/actions/cloud-database` action creates a server for the run, exports the connection strings below, and deletes the server when the job ends.
+
+| Variable | Targets | Signs in with |
+| --- | --- | --- |
+| `ServiceControl_Persistence_SqlServer_ConnectionString` | `azure-sqlserver`, `rds-sqlserver` | The admin user name and password |
+| `ServiceControl_Persistence_PostgreSql_ConnectionString` | `azure-postgresql`, `aurora-postgresql` | The admin user name and password |
+| `ServiceControl_Persistence_SqlServer_TokenConnectionString` | `azure-sqlserver` | A Microsoft Entra ID token |
+| `ServiceControl_Persistence_PostgreSql_TokenConnectionString` | `azure-postgresql`, `aurora-postgresql` | A Microsoft Entra ID token on Azure, an IAM token on Aurora |
+
+## Token authentication fixtures
+
+The suites use the password connection strings. Only the token authentication fixtures use the token connection strings, and each fixture is ignored when its variable is not set. A token fixture applies the migrations to a new schema, writes a row and reads it back, then drops the schema, all through the token login.
+
+* `azure-sqlserver` makes the service principal that `azure/login` signed in as the server's Microsoft Entra admin and keeps SQL authentication enabled. The connection string has `Authentication=Active Directory Default` and no `User ID` or `Password`. SqlClient gets the token from `DefaultAzureCredential`, which finds the Azure CLI sign-in on the runner.
+* `azure-postgresql` enables Microsoft Entra authentication alongside password authentication and makes the same service principal a Microsoft Entra admin of the server. The admin's role name is the principal's client ID. The connection string uses that client ID as its `Username` and sets `Token Authentication=AzureEntraId`.
+* `aurora-postgresql` enables IAM database authentication on the cluster and creates an `sc_iam` user that has `rds_iam` and `CREATE` on the test database. The master user must not get `rds_iam`: a user with that role can no longer sign in with a password, and the suites sign in as the master user. The connection string sets `Token Authentication=AwsRdsIam`.
+
+## AWS permissions
+
+The AWS identity that the workflow uses needs `rds-db:connect` for `sc_iam`. The file `.github/actions/cloud-database/aws-iam-policy.json` lists every permission that identity needs, this one included. The workflow does not apply the file, so add a new permission to the identity by hand before the first run that depends on it.
+
+## Running a token fixture locally
+
+Set the fixture's variable to a connection string for a server that accepts your own token login. For example, use a server where you are the Microsoft Entra admin and have signed in with `az login`.

docs/testing.md

@@ -24,7 +24,7 @@ On Windows, [PackagedRavenDBTests](../src/ServiceControlInstaller.Engine.UnitTes
 
 ## Persistence tests
 
-Persistence tests check assumptions at the persistence seam level by exercising each persister. For local setup details, see [Local testing of persistence providers](testing-persistence.md).
+Persistence tests check assumptions at the persistence seam level by exercising each persister. For local setup details, see [Local testing of persistence providers](testing-persistence.md). For the workflow that runs these suites against managed cloud databases, see [Cloud database tests](cloud-database-tests.md).
 
 The RavenDB persistence test projects contain an `IndexDefinitionApprovals` test that approves the definition of every index the persister deploys. RavenDB rebuilds an index side by side when its definition changes, and the rebuild of a large database can take days. When the test fails, record the changed indexes in the upgrade guide of the next release before you approve the new definition. An operator who locked an index must unlock it to receive the change.
 

src/Directory.Packages.props

@@ -6,7 +6,9 @@
   <ItemGroup Label="Versions for direct package references">
     <PackageVersion Include="Autofac" Version="9.3.2" />
     <PackageVersion Include="AWSSDK.CloudWatch" Version="4.0.103.3" />
+    <PackageVersion Include="AWSSDK.RDS" Version="4.0.104.2" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.102.2" />
+    <PackageVersion Include="AWSSDK.SecurityToken" Version="4.0.7.3" />
     <PackageVersion Include="Azure.Identity" Version="1.21.0" />
     <PackageVersion Include="Azure.ResourceManager.Monitor" Version="1.3.1" />
     <PackageVersion Include="Azure.ResourceManager.ServiceBus" Version="1.2.0" />