Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/webapp/app/env.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -715,6 +715,8 @@ const EnvironmentSchema = z
OBJECT_STORE_SECRET_ACCESS_KEY: z.string().optional(),
OBJECT_STORE_REGION: z.string().optional(),
OBJECT_STORE_SERVICE: z.string().default("s3"),
// IAM mode only. Set false for endpoints that reject path-style requests (AWS FIPS endpoints).
OBJECT_STORE_FORCE_PATH_STYLE: BoolEnv.default(true),

// Protocol to use for new uploads (e.g., "s3", "r2"). Data without protocol uses default provider above.
// If specified, you must configure the corresponding provider using OBJECT_STORE_{PROTOCOL}_* env vars.
Expand Down
5 changes: 5 additions & 0 deletions apps/webapp/app/v3/objectStore.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { ServiceValidationError } from "~/v3/services/common.server";
import { singleton } from "~/utils/singleton";
import { BoolEnv } from "~/utils/boolEnv";
import {
normalizeObjectStoreLogicalKeyPathname,
ObjectStoreClient,
Expand Down Expand Up @@ -182,6 +183,9 @@ function getObjectStoreConfig(protocol?: string): ObjectStoreClientConfig | unde
secretAccessKey: process.env[`${prefix}SECRET_ACCESS_KEY`] || undefined,
region: process.env[`${prefix}REGION`] || undefined,
service: process.env[`${prefix}SERVICE`] || undefined,
forcePathStyle: BoolEnv.optional().parse(
process.env[`${prefix}FORCE_PATH_STYLE`] || undefined
),
};
}

Expand All @@ -197,6 +201,7 @@ function getObjectStoreConfig(protocol?: string): ObjectStoreClientConfig | unde
secretAccessKey: env.OBJECT_STORE_SECRET_ACCESS_KEY || undefined,
region: env.OBJECT_STORE_REGION || undefined,
service: env.OBJECT_STORE_SERVICE || undefined,
forcePathStyle: env.OBJECT_STORE_FORCE_PATH_STYLE,
};
}

Expand Down
10 changes: 9 additions & 1 deletion apps/webapp/app/v3/objectStoreClient.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ type AwsSdkConfig = {
bucket: string;
baseUrl: string;
region?: string;
forcePathStyle: boolean;
};

class AwsSdkClient implements IObjectStoreClient {
Expand All @@ -95,7 +96,7 @@ class AwsSdkClient implements IObjectStoreClient {
constructor(private readonly config: AwsSdkConfig) {
this.s3Client = new S3Client({
endpoint: config.baseUrl,
forcePathStyle: true,
forcePathStyle: config.forcePathStyle,
...(config.region ? { region: config.region } : {}),
});
}
Expand Down Expand Up @@ -164,6 +165,12 @@ export type ObjectStoreClientConfig = {
secretAccessKey?: string;
region?: string;
service?: string;
/**
* IAM mode only: address objects as `<baseUrl>/<bucket>/<key>` (true, the default) or
* `<bucket>.<baseUrl host>/<key>` (false). AWS FIPS endpoints such as
* `s3-fips.us-gov-west-1.amazonaws.com` only accept the virtual-hosted form.
*/
forcePathStyle?: boolean;
};

export class ObjectStoreClient implements IObjectStoreClient {
Expand Down Expand Up @@ -199,6 +206,7 @@ export class ObjectStoreClient implements IObjectStoreClient {
bucket: config.bucket,
baseUrl: config.baseUrl,
region: config.region,
forcePathStyle: config.forcePathStyle ?? true,
}),
config.bucket
);
Expand Down
53 changes: 53 additions & 0 deletions apps/webapp/test/objectStoreClient.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { ObjectStoreClient } from "~/v3/objectStoreClient.server";

// Presigning is local SigV4 signing, so these tests need no network or containers.
// Static env credentials stand in for the IAM credential chain the client resolves at sign time.
const FIPS_ENDPOINT = "https://s3-fips.us-gov-west-1.amazonaws.com";
const BUCKET = "trigger-bucket";
const KEY = `${BUCKET}/packets/proj_123/run_456/payload.json`;

function iamClient(forcePathStyle?: boolean) {
return ObjectStoreClient.create({
baseUrl: FIPS_ENDPOINT,
bucket: BUCKET,
region: "us-gov-west-1",
...(forcePathStyle === undefined ? {} : { forcePathStyle }),
});
}

describe("ObjectStoreClient IAM-mode addressing", () => {
beforeEach(() => {
vi.stubEnv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE");
vi.stubEnv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
vi.stubEnv("AWS_SESSION_TOKEN", undefined);
vi.stubEnv("AWS_PROFILE", undefined);
});

afterEach(() => {
vi.unstubAllEnvs();
});

it("keeps path-style URLs when forcePathStyle is not set", async () => {
const url = new URL(await iamClient().presign(KEY, "PUT", 300));

expect(url.host).toBe("s3-fips.us-gov-west-1.amazonaws.com");
expect(url.pathname).toBe(`/${KEY}`);
});

it("uses virtual-hosted URLs when forcePathStyle is false", async () => {
const url = new URL(await iamClient(false).presign(KEY, "PUT", 300));

expect(url.host).toBe(`${BUCKET}.s3-fips.us-gov-west-1.amazonaws.com`);
expect(url.pathname).toBe("/packets/proj_123/run_456/payload.json");
expect(url.searchParams.get("X-Amz-Algorithm")).toBe("AWS4-HMAC-SHA256");
expect(url.searchParams.get("X-Amz-Expires")).toBe("300");
});

it("applies the same addressing to GET presigns", async () => {
const url = new URL(await iamClient(false).presign(KEY, "GET", 300));

expect(url.host).toBe(`${BUCKET}.s3-fips.us-gov-west-1.amazonaws.com`);
expect(url.pathname).toBe("/packets/proj_123/run_456/payload.json");
});
});
2 changes: 2 additions & 0 deletions docs/self-hosting/env/webapp.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,14 @@ mode: "wide"
| `OBJECT_STORE_SECRET_ACCESS_KEY` | No | — | Object store secret key (default provider). |
| `OBJECT_STORE_REGION` | No | — | Object store region (default provider). |
| `OBJECT_STORE_SERVICE` | No | s3 | Object store service (default provider). |
| `OBJECT_STORE_FORCE_PATH_STYLE` | No | true | IAM mode only (no access keys). Set `false` for virtual-hosted addressing, which AWS FIPS endpoints require. |
| `OBJECT_STORE_DEFAULT_PROTOCOL` | No | — | Protocol for new uploads (e.g. `s3`, `r2`). Enables protocol-prefixed storage. See migration guide below. |
| `OBJECT_STORE_{PROTOCOL}_BASE_URL` | No | — | Named provider base URL (replace `{PROTOCOL}`, e.g. `OBJECT_STORE_S3_BASE_URL`). |
| `OBJECT_STORE_{PROTOCOL}_ACCESS_KEY_ID` | No | — | Named provider access key. |
| `OBJECT_STORE_{PROTOCOL}_SECRET_ACCESS_KEY` | No | — | Named provider secret key. |
| `OBJECT_STORE_{PROTOCOL}_REGION` | No | — | Named provider region. |
| `OBJECT_STORE_{PROTOCOL}_SERVICE` | No | — | Named provider service. |
| `OBJECT_STORE_{PROTOCOL}_FORCE_PATH_STYLE` | No | true | Named provider path-style addressing (IAM mode only). |
| `ARTIFACTS_OBJECT_STORE_BUCKET` | No | — | Optional separate bucket for artifacts. If not set, uses main object store. |
| `ARTIFACTS_OBJECT_STORE_BASE_URL` | No | — | Optional artifacts store base URL. |
| `ARTIFACTS_OBJECT_STORE_ACCESS_KEY_ID` | No | — | Optional artifacts store access key. |
Expand Down
Loading