fix(importer): track REST Last-Modified header separately from LastUpdated

go/internal/database/datastore/models.go

@@ -162,6 +162,7 @@ type SourceRepository struct {
 	DirectoryPath           string                      `datastore:"directory_path"`
 	LastSyncedHash          string                      `datastore:"last_synced_hash"`
 	LastUpdateDate          *time.Time                  `datastore:"last_update_date"`
+	RESTHTTPLastModified    *time.Time                  `datastore:"rest_http_last_modified,omitempty"`
 	IgnorePatterns          []string                    `datastore:"ignore_patterns"`
 	Editable                bool                        `datastore:"editable"`
 	Extension               string                      `datastore:"extension"`

go/internal/database/datastore/source_repository.go

@@ -124,6 +124,7 @@ func (sr *SourceRepository) toModel() *models.SourceRepository {
 		msr.REST = &models.SourceRepoREST{
 			URL:                     sr.RESTAPIURL,
 			LastUpdated:             sr.LastUpdateDate,
+			HTTPLastModified:        sr.RESTHTTPLastModified,
 			IgnoreLastImportTime:    sr.IgnoreLastImportTime,
 			IgnoreDeletionThreshold: sr.IgnoreDeletionThreshold,
 		}
@@ -173,6 +174,7 @@ func newSourceRepositoryFromModel(r *models.SourceRepository) *SourceRepository
 	if r.REST != nil {
 		sr.RESTAPIURL = r.REST.URL
 		sr.LastUpdateDate = r.REST.LastUpdated
+		sr.RESTHTTPLastModified = r.REST.HTTPLastModified
 		sr.IgnoreLastImportTime = r.REST.IgnoreLastImportTime
 		sr.IgnoreDeletionThreshold = r.REST.IgnoreDeletionThreshold
 	}

go/internal/database/datastore/source_repository_test.go

@@ -106,7 +106,8 @@ func TestSourceRepositoryStore_Update(t *testing.T) {
 				Type: models.SourceRepositoryTypeREST,
 				REST: &models.SourceRepoREST{
 					URL:                     "http://rest.url",
-					LastUpdated:             &testTime2,
+					LastUpdated:             &testTime,
+					HTTPLastModified:        &testTime2,
 					IgnoreLastImportTime:    true,
 					IgnoreDeletionThreshold: true,
 				},
@@ -115,7 +116,8 @@ func TestSourceRepositoryStore_Update(t *testing.T) {
 				Name:                    "test-rest-repo",
 				Type:                    models.SourceRepositoryTypeREST,
 				RESTAPIURL:              "http://rest.url",
-				LastUpdateDate:          &testTime2,
+				LastUpdateDate:          &testTime,
+				RESTHTTPLastModified:    &testTime2,
 				IgnoreLastImportTime:    true,
 				IgnoreDeletionThreshold: true,
 			},
@@ -251,6 +253,29 @@ func TestSourceRepositoryStore_Get(t *testing.T) {
 	if diff := cmp.Diff(bucketRepo, gotBucket, cmpopts.EquateEmpty()); diff != "" {
 		t.Errorf("Get() Bucket repo mismatch (-want +got):\n%s", diff)
 	}
+
+	testTime2 := time.Date(2023, 10, 27, 0, 0, 0, 0, time.UTC)
+	restRepo := &models.SourceRepository{
+		Name: "test-rest-repo",
+		Type: models.SourceRepositoryTypeREST,
+		REST: &models.SourceRepoREST{
+			URL:              "http://rest.url",
+			LastUpdated:      &testTime,
+			HTTPLastModified: &testTime2,
+		},
+		GitAnalysis: &models.GitAnalysisConfig{},
+	}
+	if err := store.Update(ctx, restRepo.Name, restRepo); err != nil {
+		t.Fatalf("Failed to setup REST test data: %v", err)
+	}
+
+	gotREST, err := store.Get(ctx, "test-rest-repo")
+	if err != nil {
+		t.Fatalf("Failed to get REST repo: %v", err)
+	}
+	if diff := cmp.Diff(restRepo, gotREST, cmpopts.EquateEmpty()); diff != "" {
+		t.Errorf("Get() REST repo mismatch (-want +got):\n%s", diff)
+	}
 }
 
 func TestSourceRepositoryStore_All(t *testing.T) {

go/internal/importer/rest.go

@@ -61,14 +61,18 @@ func handleImportREST(ctx context.Context, ch chan<- WorkItem, config Config, so
 		hasUpdateTime = true
 	}
 	timeOfRun := time.Now()
-	var lastModTime time.Time
+	var httpLastModified time.Time
 	if hasUpdateTime {
 		var err error
-		lastModTime, err = checkHEAD(ctx, config, sourceRepo)
+		httpLastModified, err = checkHEAD(ctx, config, sourceRepo)
 		if err != nil {
 			return err
 		}
-		if !lastModTime.IsZero() && (lastModTime.Before(lastUpdated) || lastModTime.Equal(lastUpdated)) {
+		prevHTTPLastModified := lastUpdated
+		if sourceRepo.REST.HTTPLastModified != nil {
+			prevHTTPLastModified = *sourceRepo.REST.HTTPLastModified
+		}
+		if !httpLastModified.IsZero() && !httpLastModified.After(prevHTTPLastModified) {
 			logger.InfoContext(ctx, "No changes since last update.",
 				slog.String("source", sourceRepo.Name),
 				slog.String("url", sourceRepo.REST.URL))
@@ -92,6 +96,9 @@ func handleImportREST(ctx context.Context, ch chan<- WorkItem, config Config, so
 
 		return fmt.Errorf("failed to fetch REST API: %d for %s", resp.StatusCode, sourceRepo.REST.URL)
 	}
+	if getHTTPLastModified, err := parseHTTPLastModified(resp.Header); err == nil && !getHTTPLastModified.IsZero() {
+		httpLastModified = getHTTPLastModified
+	}
 	data, err := io.ReadAll(resp.Body)
 	if err != nil {
 		logger.ErrorContext(ctx, "Failed to read REST API response", slog.String("source", sourceRepo.Name), slog.String("url", sourceRepo.REST.URL), slog.Any("error", err))
@@ -181,10 +188,16 @@ func handleImportREST(ctx context.Context, ch chan<- WorkItem, config Config, so
 	if !maxModified.IsZero() && maxModified.Before(timeToUpdate) {
 		timeToUpdate = maxModified
 	}
-	if !lastModTime.IsZero() && lastModTime.Before(timeToUpdate) {
-		timeToUpdate = lastModTime
+	if !httpLastModified.IsZero() && httpLastModified.Before(timeToUpdate) {
+		timeToUpdate = httpLastModified
 	}
 	sourceRepo.REST.LastUpdated = &timeToUpdate
+	if !httpLastModified.IsZero() {
+		modUTC := httpLastModified.UTC()
+		sourceRepo.REST.HTTPLastModified = &modUTC
+	} else {
+		sourceRepo.REST.HTTPLastModified = nil
+	}
 	sourceRepo.REST.IgnoreLastImportTime = false
 	if err := config.SourceRepoStore.Update(ctx, sourceRepo.Name, sourceRepo); err != nil {
 		logger.ErrorContext(ctx, "Failed to update source repository", slog.Any("error", err), slog.String("source", sourceRepo.Name))
@@ -322,6 +335,19 @@ func handleDeleteREST(ctx context.Context, ch chan<- WorkItem, config Config, so
 	return nil
 }
 
+func parseHTTPLastModified(h http.Header) (time.Time, error) {
+	httpLastModified := h.Get("Last-Modified")
+	if httpLastModified == "" {
+		return time.Time{}, nil
+	}
+	t, err := http.ParseTime(httpLastModified)
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	return t.UTC(), nil
+}
+
 // checkHEAD performs a HEAD request to check for updates
 func checkHEAD(ctx context.Context, config Config, sourceRepo *models.SourceRepository) (time.Time, error) {
 	req, err := http.NewRequest(http.MethodHead, sourceRepo.REST.URL, nil)
@@ -343,16 +369,15 @@ func checkHEAD(ctx context.Context, config Config, sourceRepo *models.SourceRepo
 		return time.Time{}, nil
 	}
 
-	lastModified := resp.Header.Get("Last-Modified")
-	if lastModified == "" {
+	if resp.Header.Get("Last-Modified") == "" {
 		logger.WarnContext(ctx, "No Last-Modified header found.",
 			slog.String("source", sourceRepo.Name),
 			slog.String("url", sourceRepo.REST.URL))
 
 		return time.Time{}, nil
 	}
 
-	lastModTime, err := time.Parse(time.RFC1123, lastModified)
+	httpLastModified, err := parseHTTPLastModified(resp.Header)
 	if err != nil {
 		logger.WarnContext(ctx, "Failed to parse Last-Modified header.",
 			slog.String("source", sourceRepo.Name),
@@ -362,7 +387,7 @@ func checkHEAD(ctx context.Context, config Config, sourceRepo *models.SourceRepo
 		return time.Time{}, nil
 	}
 
-	return lastModTime, nil
+	return httpLastModified, nil
 }
 
 func handleReconcileREST(ctx context.Context, ch chan<- WorkItem, config Config, sourceRepo *models.SourceRepository, timeOfRun time.Time) error {

go/internal/importer/rest_test.go

@@ -130,8 +130,8 @@ func TestHandleImportREST(t *testing.T) {
 
 	ch := make(chan WorkItem, 10)
 
-	// Ensure that Last-Modified logic allows the fetching
-	lastModifiedHeader = "Mon, 02 Jan 2023 00:00:00 GMT" // Newer than lastUpdated
+	// Ensure that Last-Modified logic allows the fetching (and is later than maxModified)
+	lastModifiedHeader = "Mon, 02 Jan 2023 12:00:00 GMT"
 	err := handleImportREST(ctx, ch, config, sourceRepo)
 	if err != nil {
 		t.Fatalf("handleImportREST failed: %v", err)
@@ -152,16 +152,27 @@ func TestHandleImportREST(t *testing.T) {
 	if records[0].urlPath != "CVE-NEWER.json" {
 		t.Errorf("Expected urlPath=CVE-NEWER.json, got %s", records[0].urlPath)
 	}
+
+	wantLastUpdated := time.Date(2023, 1, 2, 0, 0, 0, 0, time.UTC)
+	if sourceRepo.REST.LastUpdated == nil || !sourceRepo.REST.LastUpdated.Equal(wantLastUpdated) {
+		t.Errorf("LastUpdated = %v, want %v", sourceRepo.REST.LastUpdated, wantLastUpdated)
+	}
+	wantHTTPLastModified := time.Date(2023, 1, 2, 12, 0, 0, 0, time.UTC)
+	if sourceRepo.REST.HTTPLastModified == nil || !sourceRepo.REST.HTTPLastModified.Equal(wantHTTPLastModified) {
+		t.Errorf("HTTPLastModified = %v, want %v", sourceRepo.REST.HTTPLastModified, wantHTTPLastModified)
+	}
 }
 
 func TestHandleImportREST_HEAD_NoChanges(t *testing.T) {
 	ctx := t.Context()
-	lastUpdated := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
+	// Simulate the common case where index upload Last-Modified (12:00) is newer than
+	// the latest advisory modified timestamp stored in LastUpdated (00:00).
+	lastUpdated := time.Date(2023, 1, 2, 0, 0, 0, 0, time.UTC)
+	httpLastModified := time.Date(2023, 1, 2, 12, 0, 0, 0, time.UTC)
 
-	// Server returns a Last-Modified older than LastUpdated
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == http.MethodHead {
-			w.Header().Set("Last-Modified", "Sat, 31 Dec 2022 00:00:00 GMT")
+			w.Header().Set("Last-Modified", "Mon, 02 Jan 2023 12:00:00 GMT")
 			w.WriteHeader(http.StatusOK)
 
 			return
@@ -178,8 +189,9 @@ func TestHandleImportREST_HEAD_NoChanges(t *testing.T) {
 		Name: "test-rest-repo",
 		Type: models.SourceRepositoryTypeREST,
 		REST: &models.SourceRepoREST{
-			URL:         ts.URL,
-			LastUpdated: &lastUpdated,
+			URL:              ts.URL,
+			LastUpdated:      &lastUpdated,
+			HTTPLastModified: &httpLastModified,
 		},
 	}
 

go/internal/models/source_repository.go

@@ -124,6 +124,8 @@ type SourceRepoREST struct {
 	URL string
 	// Last date recurring updates were requested.
 	LastUpdated *time.Time
+	// Last-Modified HTTP header time from the previous REST response.
+	HTTPLastModified *time.Time
 	// Ignore last import time once.
 	IgnoreLastImportTime bool
 	// Ignore deletion threshold.