Local stored response fetching (#3600)

stored_requests/backends/file_fetcher/fetcher.go

@@ -3,6 +3,7 @@ package file_fetcher
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"strings"
@@ -35,8 +36,15 @@ func (fetcher *eagerFetcher) FetchRequests(ctx context.Context, requestIDs []str
 	return storedRequests, storedImpressions, errs
 }
 
+// Fetch Responses - Implements the interface to read the stored response information from the fetcher's FileSystem, the directory name is "stored_responses"
 func (fetcher *eagerFetcher) FetchResponses(ctx context.Context, ids []string) (data map[string]json.RawMessage, errs []error) {
-	return nil, nil
+	storedRespFS, found := fetcher.FileSystem.Directories["stored_responses"]
+	if !found {
+		return nil, append(errs, errors.New(`no "stored_responses" directory found`))
+	}
+
+	data = storedRespFS.Files
+	return data, appendErrors("Response", ids, data, nil)
 }
 
 // FetchAccount fetches the host account configuration for a publisher

stored_requests/backends/file_fetcher/fetcher_test.go

@@ -12,11 +12,13 @@ import (
 )
 
 func TestFileFetcher(t *testing.T) {
+	// Load the test input files for testing
 	fetcher, err := NewFileFetcher("./test")
 	if err != nil {
 		t.Errorf("Failed to create a Fetcher: %v", err)
 	}
 
+	// Test stored request and stored imps
 	storedReqs, storedImps, errs := fetcher.FetchRequests(context.Background(), []string{"1", "2"}, []string{"some-imp"})
 	assertErrorCount(t, 0, errs)
 
@@ -25,6 +27,54 @@ func TestFileFetcher(t *testing.T) {
 	validateImp(t, storedImps)
 }
 
+func TestStoredResponseFileFetcher(t *testing.T) {
+	// grab the fetcher that do not have /test/stored_responses/stored_responses FS directory
+	directoryNotExistfetcher, err := NewFileFetcher("./test/stored_responses")
+	if err != nil {
+		t.Errorf("Failed to create a Fetcher: %v", err)
+	}
+
+	// we should receive 1 error since we do not have "stored_responses" directory in ./test/stored_responses
+	_, errs := directoryNotExistfetcher.FetchResponses(context.Background(), []string{})
+	assertErrorCount(t, 1, errs)
+
+	// grab the fetcher that has /test/stored_responses FS directory
+	fetcher, err := NewFileFetcher("./test")
+	if err != nil {
+		t.Errorf("Failed to create a Fetcher: %v", err)
+	}
+
+	// Test stored responses, we have 3 stored responses in ./test/stored_responses
+	storedResps, errs := fetcher.FetchResponses(context.Background(), []string{"bar", "escaped", "does_not_exist"})
+	// expect 1 error since we do not have "does_not_exist" stored response file from ./test
+	assertErrorCount(t, 1, errs)
+
+	validateStoredResponse[map[string]string](t, storedResps, "bar", func(val map[string]string) error {
+		if len(val) != 1 {
+			return fmt.Errorf("Unexpected value length. Expected %d, Got %d", 1, len(val))
+		}
+
+		data, hadKey := val["test"]
+		if !hadKey {
+			return fmt.Errorf(`missing key "test" in the value`)
+		}
+
+		expectedVal := "bar"
+		if data != expectedVal {
+			return fmt.Errorf(`Bad value for key "test". Expected "%s", Got "%s"`, expectedVal, data)
+		}
+		return nil
+	})
+
+	validateStoredResponse[string](t, storedResps, "escaped", func(val string) error {
+		expectedVal := `esca"ped`
+		if val != expectedVal {
+			return fmt.Errorf(`Bad data. Expected "%v", Got "%s"`, expectedVal, val)
+		}
+		return nil
+	})
+}
+
 func TestAccountFetcher(t *testing.T) {
 	fetcher, err := NewFileFetcher("./test")
 	assert.NoError(t, err, "Failed to create test fetcher")
@@ -179,3 +229,20 @@ func TestCategoriesFetcherNoCategoriesFile(t *testing.T) {
 	assert.Equal(t, fmt.Errorf("Unable to find mapping file for adserver: 'test', publisherId: 'not_exists'"),
 		fetchingErr, "Categories were loaded incorrectly")
 }
+
+// validateStoredResponse - reusable function in the stored response test to verify the actual data read from the fetcher
+func validateStoredResponse[T any](t *testing.T, storedInfo map[string]json.RawMessage, id string, verifyFunc func(outputVal T) error) {
+	storedValue, hasID := storedInfo[id]
+	if !hasID {
+		t.Fatalf(`Expected stored response data to have id: "%s"`, id)
+	}
+
+	var unmarshalledValue T
+	if err := jsonutil.UnmarshalValid(storedValue, &unmarshalledValue); err != nil {
+		t.Errorf(`Failed to unmarshal stored response data of id "%s": %v`, id, err)
+	}
+
+	if err := verifyFunc(unmarshalledValue); err != nil {
+		t.Errorf(`Bad data in stored response of id: "%s": %v`, id, err)
+	}
+}

stored_requests/backends/file_fetcher/test/stored_responses/bar.json

@@ -0,0 +1,4 @@
+{
+    "test": "bar"
+}
+

stored_requests/backends/file_fetcher/test/stored_responses/escaped.json

@@ -0,0 +1 @@
+"esca\"ped"