Host generic files and share one push path across hosted ecosystems

cmd/proxy/hosted.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -9,20 +11,24 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 	"text/tabwriter"
 	"time"
+
+	"github.com/git-pkgs/proxy/internal/handler"
 )
 
 const (
 	defaultProxyURL     = "http://localhost:8080"
 	hostedTokenEnv      = "PROXY_HOSTED_TOKEN"
 	hostedClientTimeout = 10 * time.Minute
 
-	pushArgCount   = 4 // ecosystem, name, version, file
-	listArgCount   = 2 // ecosystem, name
-	deleteArgCount = 3 // ecosystem, name, version
-	tablePadding   = 2
+	pushArgCount      = 4 // ecosystem, name, version, file
+	listArgCount      = 2 // ecosystem, name
+	deleteArgCount    = 3 // ecosystem, name, version
+	deleteMaxArgCount = 4 // ecosystem, name, version, file for generic files
+	tablePadding      = 2
 )
 
 // hostedArtifact mirrors the JSON the /api/hosted endpoints return.
@@ -69,15 +75,16 @@ func newHostedCommand(name, usage, summary string, getenv func(string) string) h
 	return cmd
 }
 
-// parse parses flags and checks for exactly want positional arguments.
-func (c hostedCommand) parse(args []string, want int) ([]string, error) {
+// parse parses flags and checks for between minArgs and maxArgs positional
+// arguments.
+func (c hostedCommand) parse(args []string, minArgs, maxArgs int) ([]string, error) {
 	if err := c.flags.Parse(args); err != nil {
 		if errors.Is(err, flag.ErrHelp) {
 			c.flags.Usage()
 		}
 		return nil, fmt.Errorf("%w\nusage: %s", err, c.usage)
 	}
-	if c.flags.NArg() != want {
+	if n := c.flags.NArg(); n < minArgs || n > maxArgs {
 		return nil, fmt.Errorf("usage: %s", c.usage)
 	}
 	return c.flags.Args(), nil
@@ -105,7 +112,7 @@ func hostedToken(getenv func(string) string) (string, error) {
 func runPush(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
 	cmd := newHostedCommand("push", "proxy hosted push [-url URL] <ecosystem> <name> <version> <file>",
 		"Push a package archive to the proxy", getenv)
-	positional, err := cmd.parse(args, pushArgCount)
+	positional, err := cmd.parse(args, pushArgCount, pushArgCount)
 	if err != nil {
 		return err
 	}
@@ -121,12 +128,32 @@ func runPush(args []string, getenv func(string) string, stdout io.Writer, client
 	}
 	defer func() { _ = file.Close() }()
 
-	req, err := http.NewRequest(http.MethodPut, cmd.endpoint(ecosystem, name, version), file)
+	// The proxy checks the upload against this, so a truncated or corrupted
+	// transfer is refused rather than stored.
+	digest := sha256.New()
+	size, err := io.Copy(digest, file)
+	if err != nil {
+		return err
+	}
+	if _, err := file.Seek(0, io.SeekStart); err != nil {
+		return err
+	}
+
+	segments := []string{version}
+	if handler.HostedFileInPath(ecosystem) {
+		segments = append(segments, filepath.Base(path))
+	}
+	endpoint := cmd.endpoint(ecosystem, name, segments...) + "?sha256=" + hex.EncodeToString(digest.Sum(nil))
+	req, err := http.NewRequest(http.MethodPut, endpoint, file)
 	if err != nil {
 		return err
 	}
+	req.ContentLength = size
 	req.Header.Set("Authorization", "Bearer "+token)
 	req.Header.Set("Content-Type", "application/octet-stream")
+	// The proxy refuses an existing file or a bad token before reading the
+	// body, so the file is only uploaded once it would be accepted.
+	req.Header.Set("Expect", "100-continue")
 
 	var pushed hostedPushResponse
 	if err := doHosted(client, req, http.StatusCreated, &pushed); err != nil {
@@ -139,7 +166,7 @@ func runPush(args []string, getenv func(string) string, stdout io.Writer, client
 
 func runList(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
 	cmd := newHostedCommand("list", "proxy hosted list [-url URL] <ecosystem> <name>", "List pushed package archives", getenv)
-	positional, err := cmd.parse(args, listArgCount)
+	positional, err := cmd.parse(args, listArgCount, listArgCount)
 	if err != nil {
 		return err
 	}
@@ -161,16 +188,20 @@ func runList(args []string, getenv func(string) string, stdout io.Writer, client
 }
 
 func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error {
-	cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] <ecosystem> <name> <version>", "Delete a pushed version", getenv)
-	positional, err := cmd.parse(args, deleteArgCount)
+	cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] <ecosystem> <name> <version> [file]",
+		"Delete a pushed version, or one generic file of it", getenv)
+	positional, err := cmd.parse(args, deleteArgCount, deleteMaxArgCount)
 	if err != nil {
 		return err
 	}
+	if len(positional) > deleteArgCount && !handler.HostedFileInPath(positional[0]) {
+		return fmt.Errorf("only generic files can be deleted one at a time\nusage: %s", cmd.usage)
+	}
 	token, err := hostedToken(getenv)
 	if err != nil {
 		return err
 	}
-	req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2]), nil)
+	req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2:]...), nil)
 	if err != nil {
 		return err
 	}
@@ -179,7 +210,11 @@ func runDelete(args []string, getenv func(string) string, stdout io.Writer, clie
 	if err := doHosted(client, req, http.StatusNoContent, nil); err != nil {
 		return err
 	}
-	_, err = fmt.Fprintf(stdout, "deleted %s %s@%s\n", positional[0], positional[1], positional[2])
+	deleted := positional[1] + "@" + positional[2]
+	if len(positional) > deleteArgCount {
+		deleted += " " + positional[deleteArgCount]
+	}
+	_, err = fmt.Fprintf(stdout, "deleted %s %s\n", positional[0], deleted)
 	return err
 }
 
@@ -210,7 +245,7 @@ func doHosted(client *http.Client, req *http.Request, wantStatus int, out any) e
 const hostedUsage = "usage: proxy hosted <push|list|delete> [flags] ...\n\n" +
 	"  push     Push a package archive to the proxy\n" +
 	"  list     List pushed package archives\n" +
-	"  delete   Delete a pushed version"
+	"  delete   Delete a pushed version, or one generic file of it"
 
 type hostedSubcommand func([]string, func(string) string, io.Writer, *http.Client) error
 

cmd/proxy/hosted_test.go

@@ -12,7 +12,8 @@ import (
 )
 
 type recordedRequest struct {
-	method, path, query, authorization, body string
+	method, path, query, authorization, body, expect string
+	contentLength                                    int64
 }
 
 func hostedTestServer(t *testing.T, status int, response string) (*httptest.Server, *recordedRequest) {
@@ -26,6 +27,8 @@ func hostedTestServer(t *testing.T, status int, response string) (*httptest.Serv
 			query:         r.URL.RawQuery,
 			authorization: r.Header.Get("Authorization"),
 			body:          string(body),
+			expect:        r.Header.Get("Expect"),
+			contentLength: r.ContentLength,
 		}
 		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(status)
@@ -62,14 +65,34 @@ func TestRunPush(t *testing.T) {
 	if recorded.method != http.MethodPut || recorded.path != "/api/hosted/npm/@types/node/20.1.0" {
 		t.Errorf("request = %s %s", recorded.method, recorded.path)
 	}
-	if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "" {
+	// sha256 of "tarball", so the proxy can check the upload.
+	const tarballSHA256 = "db4b4d0d1cb480bf9aeea253771c00febe627f236765fa37d6a5614f079a3aa0"
+	if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "sha256="+tarballSHA256 {
 		t.Errorf("authorization/body/query = %q/%q/%q", recorded.authorization, recorded.body, recorded.query)
 	}
+	if recorded.expect != "100-continue" || recorded.contentLength != int64(len("tarball")) {
+		t.Errorf("expect/content length = %q/%d, want 100-continue and the file size", recorded.expect, recorded.contentLength)
+	}
 	if out := stdout.String(); !strings.Contains(out, "node-20.1.0.tgz") || !strings.Contains(out, "abc123") {
 		t.Errorf("output = %q, want filename and sha256", out)
 	}
 }
 
+func TestRunPushGenericNamesTheFile(t *testing.T) {
+	srv, recorded := hostedTestServer(t, http.StatusCreated,
+		`{"ecosystem":"generic","name":"tools","version":"1.2.0","filename":"tool-linux-arm64.tar.gz","sha256":"abc123","size":7}`)
+	archive := writeArchive(t, "tool-linux-arm64.tar.gz", "tarball")
+
+	err := runPush([]string{"-url", srv.URL, "generic", "tools", "1.2.0", archive},
+		hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
+	if err != nil {
+		t.Fatalf("runPush: %v", err)
+	}
+	if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" {
+		t.Errorf("path = %q, want the file name appended", recorded.path)
+	}
+}
+
 func TestRunPushErrors(t *testing.T) {
 	srv, _ := hostedTestServer(t, http.StatusBadRequest, `{"code":"BAD_REQUEST","message":"pushing packages is not supported for this ecosystem: cargo"}`)
 	archive := writeArchive(t, "serde-1.0.200.crate", "crate")
@@ -141,6 +164,36 @@ func TestRunDelete(t *testing.T) {
 	}
 }
 
+func TestRunDeleteOneFile(t *testing.T) {
+	srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
+	var stdout bytes.Buffer
+
+	err := runDelete([]string{"-url", srv.URL, "generic", "tools", "1.2.0", "tool-linux-arm64.tar.gz"},
+		hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client())
+	if err != nil {
+		t.Fatalf("runDelete: %v", err)
+	}
+	if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" {
+		t.Errorf("path = %q", recorded.path)
+	}
+	if out := stdout.String(); out != "deleted generic tools@1.2.0 tool-linux-arm64.tar.gz\n" {
+		t.Errorf("output = %q", out)
+	}
+}
+
+func TestRunDeleteOneFileOnlyForGeneric(t *testing.T) {
+	srv, recorded := hostedTestServer(t, http.StatusNoContent, "")
+
+	err := runDelete([]string{"-url", srv.URL, "npm", "@acme", "widget", "1.0.0"},
+		hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client())
+	if err == nil || !strings.Contains(err.Error(), "only generic files") {
+		t.Errorf("npm delete with a fourth argument error = %v, want a refusal", err)
+	}
+	if recorded.method != "" {
+		t.Errorf("refused delete sent %s %s", recorded.method, recorded.path)
+	}
+}
+
 func TestRunHostedDispatchesSubcommands(t *testing.T) {
 	srv, recorded := hostedTestServer(t, http.StatusOK, `{"ecosystem":"npm","name":"request","artifacts":[]}`)
 

docs/configuration.md

@@ -1076,7 +1076,7 @@ Set to `"0"` to disable the timeout entirely (requests then rely only on the ser
 
 ## Hosted Packages
 
-Operators can push a package archive into the proxy under a name and version. The proxy serves it in place of the upstream archive with the same version, or in addition to the upstream's versions. Use it to ship a patched build of a dependency without running a separate registry.
+Operators can push a package archive into the proxy under a name and version. The proxy serves it in place of the upstream archive with the same version, or in addition to the upstream's versions. Use it to ship a patched build of a dependency without running a separate registry. Files that aren't packages, such as prebuilt binaries other builds download, can be pushed as [generic files](#generic-files).
 
 The `/api/hosted` endpoints are disabled by default. Because a pushed version replaces the upstream one for every client, enabling them requires a token:
 
@@ -1093,7 +1093,7 @@ hosted:
 |--------|-------------|-------------|
 | `hosted.enabled` | `PROXY_HOSTED_ENABLED` | Register the `/api/hosted` endpoints (default `false`) |
 | `hosted.token` | `PROXY_HOSTED_TOKEN` | Bearer token required to push and delete; required when enabled, supports `${VAR}` |
-| `hosted.max_upload_size` | `PROXY_HOSTED_MAX_UPLOAD_SIZE` | Maximum size of one pushed archive (default `100MB`) |
+| `hosted.max_upload_size` | `PROXY_HOSTED_MAX_UPLOAD_SIZE` | Maximum size of one pushed archive or file (default `100MB`) |
 | `hosted.npm_scopes` | `PROXY_HOSTED_NPM_SCOPES` | npm scopes served from pushed packages only (comma-separated in the environment) |
 | `hosted.composer_vendors` | `PROXY_HOSTED_COMPOSER_VENDORS` | Composer vendors served from pushed packages only |
 
@@ -1114,13 +1114,17 @@ hosted:
    ```bash
    curl -X PUT --data-binary @request-2.88.3.tgz \
      -H "Authorization: Bearer $PROXY_HOSTED_TOKEN" \
-     https://proxy.example.com/api/hosted/npm/request/2.88.3
+     "https://proxy.example.com/api/hosted/npm/request/2.88.3?sha256=$(sha256sum request-2.88.3.tgz | cut -d' ' -f1)"
    ```
 
+   The `sha256` query parameter is optional. When it is set, a push whose bytes don't match is refused with 400. `proxy hosted push` always sends it.
+
 3. Install it as usual, e.g. `npm install request@2.88.3`. The version is listed in the package metadata, so ranges like `^2.88.0` resolve to it too.
 4. Check what is pushed: `proxy hosted list npm request` or `GET /api/hosted/npm/request`.
 5. Remove it again: `proxy hosted delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again.
 
+A pushed file is never replaced: pushing the same version again answers 409, also under a differently cased Composer name, before the body is uploaded when the client sends `Expect: 100-continue` (as `proxy hosted push` does). Clients that pinned the file by its hash would otherwise get different bytes under the same version. To push it again, delete it first.
+
 ### Supported ecosystems
 
 Download handlers look archives up by the filename clients request, so a push is stored under that name:
@@ -1129,10 +1133,11 @@ Download handlers look archives up by the filename clients request, so a push is
 |-----------|------|----------|
 | `npm` | `{name}` or `@{scope}/{name}` | `{name}-{version}.tgz`, without the scope |
 | `composer` | `{vendor}/{package}` | `{package}-{version}.zip` |
+| `generic` | `{name}`, one path segment | named in the push path, see [Generic files](#generic-files) |
 
 Other ecosystems are not supported yet.
 
-The archive must contain the package manifest: `package.json` in the top directory of an npm tarball, or `composer.json` at the root or in the single top directory of a Composer zip. Its name, and its version when it has one, must match the push. Otherwise the push is refused with 400.
+npm and Composer archives must contain the package manifest: `package.json` in the top directory of an npm tarball, or `composer.json` at the root or in the single top directory of a Composer zip. Its name, and its version when it has one, must match the push. Otherwise the push is refused with 400.
 
 ### Package metadata
 
@@ -1157,6 +1162,26 @@ For private packages:
 
 Other packages, including ones in other scopes and vendors, are proxied as usual.
 
+### Generic files
+
+Files that aren't packages, such as a prebuilt binary that other builds download and check against a pinned sha256, are pushed to the `generic` ecosystem. They have no manifest; the name, version and filename come from the path, and a version can hold several files:
+
+```bash
+proxy hosted push generic tools 1.2.0 ./tool-linux-arm64.tar.gz
+# or
+curl -X PUT --data-binary @tool-linux-arm64.tar.gz \
+  -H "Authorization: Bearer $PROXY_HOSTED_TOKEN" \
+  https://proxy.example.com/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz
+```
+
+They are downloaded without a token from `/generic/hosted/{name}/{version}/{file}`, with the file's sha256 as `ETag`:
+
+```bash
+curl -O https://proxy.example.com/generic/hosted/tools/1.2.0/tool-linux-arm64.tar.gz
+```
+
+Generic files never come from an upstream: anything not pushed answers 404. While hosting is enabled, `hosted` can't be used as an `upstream.generic` name. `proxy hosted list generic tools` lists all versions and files. `proxy hosted delete generic tools 1.2.0` removes a version with all its files, and `proxy hosted delete generic tools 1.2.0 tool-linux-arm64.tar.gz` removes only that file.
+
 ### Storage and eviction
 
 Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones.

docs/swagger/docs.go

@@ -230,7 +230,7 @@ const docTemplate = `{
                         "BearerAuth": []
                     }
                 ],
-                "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted.",
+                "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted. Generic files are pushed to /api/hosted/generic/{name}/{version}/{file} and served at /generic/hosted/{name}/{version}/{file}. A file that was already pushed is refused with 409; delete it first to push it again.",
                 "consumes": [
                     "application/octet-stream"
                 ],
@@ -244,7 +244,7 @@ const docTemplate = `{
                 "parameters": [
                     {
                         "type": "string",
-                        "description": "Ecosystem (npm or composer)",
+                        "description": "Ecosystem (npm, composer or generic)",
                         "name": "ecosystem",
                         "in": "path",
                         "required": true
@@ -262,6 +262,12 @@ const docTemplate = `{
                         "name": "version",
                         "in": "path",
                         "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Expected sha256 of the body, hex-encoded; a mismatch is refused",
+                        "name": "sha256",
+                        "in": "query"
                     }
                 ],
                 "responses": {
@@ -289,6 +295,12 @@ const docTemplate = `{
                             "$ref": "#/definitions/server.ErrorResponse"
                         }
                     },
+                    "409": {
+                        "description": "Conflict",
+                        "schema": {
+                            "$ref": "#/definitions/server.ErrorResponse"
+                        }
+                    },
                     "413": {
                         "description": "Request Entity Too Large",
                         "schema": {
@@ -303,7 +315,7 @@ const docTemplate = `{
                         "BearerAuth": []
                     }
                 ],
-                "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again.",
+                "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again. For generic files, /api/hosted/generic/{name}/{version}/{file} removes only that file.",
                 "tags": [
                     "hosted"
                 ],