fix 743 multi file and 744 debsign support

Dockerfile

@@ -12,7 +12,7 @@ RUN addgroup --gid 10001 app \
       echo 'deb http://deb.debian.org/debian buster-backports main' > /etc/apt/sources.list.d/buster-backports.list && \
       apt update && \
       apt -y upgrade && \
-      apt -y install libltdl-dev gpg libncurses5 && \
+      apt -y install libltdl-dev gpg libncurses5 devscripts && \
       apt -y install -t buster-backports apksigner && \
       apt-get clean
 
@@ -27,6 +27,7 @@ ADD version.json /app
 RUN cd /app/src/autograph && go install .
 
 RUN cd /app/src/autograph/tools/autograph-monitor && go build -o /go/bin/autograph-monitor .
+RUN cd /app/src/autograph/tools/autograph-client && go build -o /go/bin/autograph-client .
 
 USER app
 WORKDIR /app

Makefile

@@ -83,6 +83,10 @@ showcoverage: test
 generate:
 	go generate
 
+gpg-test-clean:
+	rm -rf ~/.gnupg /tmp/autograph_gpg2*
+	killall gpg-agent
+
 # image build order:
 #
 # app -> {app-hsm,monitor}

bin/run_integration_tests.sh

@@ -68,6 +68,16 @@ docker-compose run \
 	       --entrypoint ./integration_test_api.sh \
 	       app-hsm
 
+echo "checking gpg signing"
+docker-compose run \
+	       --rm \
+	       --user 0 \
+	       -e AUTOGRAPH_URL=http://app:8000 \
+	       --workdir /app/src/autograph/tools/autograph-client \
+	       --entrypoint ./integration_test_gpg2_signer.sh \
+	       app
+# TODO(GH-785): add HSM support for GPG signing keys and test here
+
 echo "checking XPI signing"
 docker-compose run \
 	       --rm \
@@ -94,4 +104,4 @@ docker-compose run \
 	       --workdir /app/src/autograph/tools/autograph-client \
 	       --entrypoint ./build_test_apks.sh \
 	       app
-# TODO: add HSM support for APK signing keys and test here
+# TODO(GH-381): add HSM support for APK signing keys and test here

docs/endpoints.md

@@ -82,6 +82,96 @@ Each signature response contains the following fields:
     format. Each signer uses a different format, so refer to their
     documentation for more information.
 
+## /sign/files
+
+### Request
+
+Request to sign multiple files. The files to sign are passed in
+the request body using the JSON format described below.
+
+The request body is an array of signature requests, to allow for
+batching signatures into a single API request. The parameters are:
+
+-   **keyid**: allows the caller to specify a key to sign the data with.
+    This parameter is optional, and Autograph will pick a key based on
+    the caller\'s permission if omitted.
+-   **options**: a JSON object used to pass signer-specific options in
+    the request. Refer to the documentation of each signer to find out
+    which options they accept.
+-   **files**: an array of dicts of file name and base64 encoded content to sign. For example:
+
+```json
+[
+    {
+        "content": "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAATAAAAQW5kcm9pZE1hbmlmZXN0LnhtbKSYS2ybx7XHf0PqbVmW4...BwAACigAAAAA",
+        "name": "sphinx_1.7.2-1.dsc"
+    },
+    {
+        "content": "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAATAAAAQW5kcm9pZE1hbmlmZXN0LnhtbKSYS2ybx7XHf0PqbVmW4...BwAACigAAAAA",
+        "name": "sphinx_1.7.2-1.changes"
+    }
+]
+```
+
+The number of files must be between 1 and 32 inclusive. All characters
+in a file name must be an alphanumeric character, dash, underscore or
+dot. The file name must start with an alphanumeric character and dots
+cannot occur next to each other in the file name.
+
+example:
+
+``` bash
+POST /sign/files
+Host: autograph.example.net
+Content-type: application/json
+Authorization: Hawk id="alice", mac="756lSgQEYLoc6V0Uv2wS8pRg/h+4WFUVKWQynCFvY8Y=", ts="1524487134", nonce="MrpGL35q", hash="9m3WhtGQDuHermi5fDYBGJlOqNeK5B3nk0lKreZ+YSw=", ext="933126753"
+
+[
+    {
+      "files":[
+         {
+             "content": "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAATAAAAQW5kcm9pZE1hbmlmZXN0LnhtbKSYS2ybx7XHf0PqbVmW4...BwAACigAAAAA",
+             "name": "sphinx_1.7.2-1.dsc"
+         },
+         {
+             "content": "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAATAAAAQW5kcm9pZE1hbmlmZXN0LnhtbKSYS2ybx7XHf0PqbVmW4...BwAACigAAAAA",
+             "name": "sphinx_1.7.2-1.changes"
+         }
+      ]
+    },
+    {
+      "files":[
+         {
+             "content": "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAATAAAAQW5kcm9pZE1hbmlmZXN0LnhtbKSYS2ybx7XHf0PqbVmW4...BwAACigAAAAA",
+             "name": "sphinx_1.7.2-1.dsc"
+         },
+         {
+             "content": "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAATAAAAQW5kcm9pZE1hbmlmZXN0LnhtbKSYS2ybx7XHf0PqbVmW4...BwAACigAAAAA",
+             "name": "sphinx_1.7.2-1.changes"
+         },
+      ],
+      "keyid":"randompgp-debsign",
+      "options":null
+    }
+]
+```
+
+### Response
+
+A successful request returns `201 Created` with a response body
+containing all signed files encoded in JSON. The ordering of the
+response array is identical to the request array, such that signing
+request 0 maps to signing response 0, etc.
+
+The response format is the same as `/sign/data` except
+instead of the `signature` field autograph returns the
+field:
+
+-   `signed_files` an array of dicts of file name and base64 encoded
+    signed file content as described in the request section. Each
+    signer uses a different format, so refer to their documentation
+    for more information.
+
 ## /sign/file
 
 ### Request

formats/rest.go

@@ -1,22 +1,32 @@
 package formats
 
+// SigningFile is a file to sign when included in a request to sign
+// multiple files or a signed file when included in a response to
+// signing multiple files
+type SigningFile struct {
+	Name    string `json:"name"`
+	Content string `json:"content"`
+}
+
 // SignatureRequest is sent by a client to request a signature on input data
 type SignatureRequest struct {
-	Input   string `json:"input"`
-	KeyID   string `json:"keyid,omitempty"`
+	Input   string        `json:"input"`
+	Files   []SigningFile `json:"files,omitempty"`
+	KeyID   string        `json:"keyid,omitempty"`
 	Options interface{}
 }
 
 // SignatureResponse is returned by autograph to a client with
 // a signature computed on input data
 type SignatureResponse struct {
-	Ref        string      `json:"ref"`
-	Type       string      `json:"type"`
-	Mode       string      `json:"mode"`
-	SignerID   string      `json:"signer_id"`
-	PublicKey  string      `json:"public_key"`
-	Signature  string      `json:"signature,omitempty"`
-	SignedFile string      `json:"signed_file,omitempty"`
-	X5U        string      `json:"x5u,omitempty"`
-	SignerOpts interface{} `json:"signer_opts,omitempty"`
+	Ref         string        `json:"ref"`
+	Type        string        `json:"type"`
+	Mode        string        `json:"mode"`
+	SignerID    string        `json:"signer_id"`
+	PublicKey   string        `json:"public_key"`
+	Signature   string        `json:"signature,omitempty"`
+	SignedFile  string        `json:"signed_file,omitempty"`
+	SignedFiles []SigningFile `json:"signed_files,omitempty"`
+	X5U         string        `json:"x5u,omitempty"`
+	SignerOpts  interface{}   `json:"signer_opts,omitempty"`
 }

handlers.go

@@ -25,6 +25,16 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+const (
+	// MinNamedFiles is the minimum number of named files a single
+	// multi-file signing request can include
+	MinNamedFiles = 1
+
+	// MaxNamedFiles is the maximum number of named files a single
+	// multi-file signing request can include
+	MaxNamedFiles = 32
+)
+
 // heartbeatConfig configures the heartbeat handler. It sets timeouts
 // for each backing service to check.
 //
@@ -47,18 +57,20 @@ func hashSHA256AsHex(toHash []byte) string {
 	return fmt.Sprintf("%X", h.Sum(nil))
 }
 
-func logSigningRequestFailure(sigreq formats.SignatureRequest, sigresp formats.SignatureResponse, rid, userid, inputHash string, starttime time.Time, err error) {
+func logSigningRequestFailure(sigreq formats.SignatureRequest, sigresp formats.SignatureResponse, rid, userid, inputHash string, inputHashes []string, starttime time.Time, err error) {
 	log.WithFields(log.Fields{
-		"rid":         rid,
-		"options":     sigreq.Options,
-		"mode":        sigresp.Mode,
-		"ref":         sigresp.Ref,
-		"type":        sigresp.Type,
-		"signer_id":   sigresp.SignerID,
-		"input_hash":  inputHash,
-		"output_hash": nil,
-		"user_id":     userid,
-		"t":           int32(time.Since(starttime) / time.Millisecond), //  request processing time in ms
+		"rid":           rid,
+		"options":       sigreq.Options,
+		"mode":          sigresp.Mode,
+		"ref":           sigresp.Ref,
+		"type":          sigresp.Type,
+		"signer_id":     sigresp.SignerID,
+		"input_hash":    inputHash,
+		"input_hashes":  inputHashes,
+		"output_hash":   nil,
+		"output_hashes": nil,
+		"user_id":       userid,
+		"t":             int32(time.Since(starttime) / time.Millisecond), //  request processing time in ms
 	}).Info(fmt.Sprintf("signing operation failed with error: %v", err))
 }
 
@@ -121,7 +133,21 @@ func (a *autographer) handleSignature(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	for i, sigreq := range sigreqs {
-		if sigreq.Input == "" {
+		if r.URL.RequestURI() == "/sign/files" {
+			if sigreq.Input != "" {
+				httpError(w, r, http.StatusBadRequest, fmt.Sprintf("input should be empty in sign files signature request %d", i))
+			}
+			if sigreq.Files == nil {
+				httpError(w, r, http.StatusBadRequest, fmt.Sprintf("missing Files in sign files signature request %d", i))
+			}
+			if len(sigreq.Files) < MinNamedFiles {
+				httpError(w, r, http.StatusBadRequest, "Did not receive enough files to sign. Need at least %d", MinNamedFiles)
+				return
+			} else if len(sigreq.Files) > MaxNamedFiles {
+				httpError(w, r, http.StatusBadRequest, "Received too many files to sign (max is %d)", MaxNamedFiles)
+				return
+			}
+		} else if sigreq.Input == "" {
 			httpError(w, r, http.StatusBadRequest, fmt.Sprintf("missing input in signature request %d", i))
 		}
 	}
@@ -135,17 +161,34 @@ func (a *autographer) handleSignature(w http.ResponseWriter, r *http.Request) {
 	// the signature is then encoded appropriately, and added to the response slice
 	for i, sigreq := range sigreqs {
 		var (
-			input                 []byte
-			sig                   signer.Signature
-			signedfile            []byte
-			inputHash, outputHash string
+			input                     []byte
+			unsignedNamedFiles        []signer.NamedUnsignedFile
+			sig                       signer.Signature
+			signedfile                []byte
+			signedfiles               []signer.NamedSignedFile
+			inputHash, outputHash     string
+			inputHashes, outputHashes []string
 		)
 
-		// Decode the base64 input data
-		input, err = base64.StdEncoding.DecodeString(sigreq.Input)
-		if err != nil {
-			httpError(w, r, http.StatusBadRequest, "%v", err)
-			return
+		if r.URL.RequestURI() == "/sign/files" {
+			for i, inputFile := range sigreq.Files {
+				log.Debugf("base64 decoding file %d", i)
+				unsignedNamedFile, err := signer.NewNamedUnsignedFile(inputFile)
+				if err != nil {
+					httpError(w, r, http.StatusBadRequest, "%q", err)
+					return
+				}
+				log.Debugf("base64 decoded unsigned named file %d: %s", i, unsignedNamedFile.Name)
+				unsignedNamedFiles = append(unsignedNamedFiles, *unsignedNamedFile)
+			}
+			log.Debugf("signing %d unsigned named files", len(unsignedNamedFiles))
+		} else {
+			// Decode the base64 input data
+			input, err = base64.StdEncoding.DecodeString(sigreq.Input)
+			if err != nil {
+				httpError(w, r, http.StatusBadRequest, "%v", err)
+				return
+			}
 		}
 
 		// returns an error if the signer is not found or if
@@ -179,7 +222,7 @@ func (a *autographer) handleSignature(w http.ResponseWriter, r *http.Request) {
 
 			sig, err = hashSigner.SignHash(input, sigreq.Options)
 			if err != nil {
-				logSigningRequestFailure(sigreq, sigresps[i], rid, userid, inputHash, starttime, err)
+				logSigningRequestFailure(sigreq, sigresps[i], rid, userid, inputHash, inputHashes, starttime, err)
 				httpError(w, r, http.StatusInternalServerError, "signing request %s failed with error: %v", sigresps[i].Ref, err)
 				return
 			}
@@ -200,7 +243,7 @@ func (a *autographer) handleSignature(w http.ResponseWriter, r *http.Request) {
 
 			sig, err = dataSigner.SignData(input, sigreq.Options)
 			if err != nil {
-				logSigningRequestFailure(sigreq, sigresps[i], rid, userid, inputHash, starttime, err)
+				logSigningRequestFailure(sigreq, sigresps[i], rid, userid, inputHash, inputHashes, starttime, err)
 				httpError(w, r, http.StatusInternalServerError, "signing request %s failed with error: %v", sigresps[i].Ref, err)
 				return
 			}
@@ -221,24 +264,47 @@ func (a *autographer) handleSignature(w http.ResponseWriter, r *http.Request) {
 
 			signedfile, err = fileSigner.SignFile(input, sigreq.Options)
 			if err != nil {
-				logSigningRequestFailure(sigreq, sigresps[i], rid, userid, inputHash, starttime, err)
+				logSigningRequestFailure(sigreq, sigresps[i], rid, userid, inputHash, inputHashes, starttime, err)
 				httpError(w, r, http.StatusInternalServerError, "signing request %s failed with error: %v", sigresps[i].Ref, err)
 				return
 			}
 			sigresps[i].SignedFile = base64.StdEncoding.EncodeToString(signedfile)
 			outputHash = hashSHA256AsHex(signedfile)
+		case "/sign/files":
+			multiFileSigner, ok := requestedSigner.(signer.MultipleFileSigner)
+			if !ok {
+				httpError(w, r, http.StatusBadRequest, "requested signer %q does not implement multiple file signing", requestedSignerConfig.ID)
+				return
+			}
+			// calculate a hash of the input files to log
+			for _, inputFile := range unsignedNamedFiles {
+				inputHashes = append(inputHashes, hashSHA256AsHex(inputFile.Bytes))
+			}
+
+			signedfiles, err = multiFileSigner.SignFiles(unsignedNamedFiles, sigreq.Options)
+			if err != nil {
+				logSigningRequestFailure(sigreq, sigresps[i], rid, userid, inputHash, inputHashes, starttime, err)
+				httpError(w, r, http.StatusInternalServerError, "signing request %s failed with error: %v", sigresps[i].Ref, err)
+				return
+			}
+			for _, signedFile := range signedfiles {
+				outputHashes = append(outputHashes, hashSHA256AsHex(signedFile.Bytes))
+				sigresps[i].SignedFiles = append(sigresps[i].SignedFiles, *signedFile.RESTSigningFile())
+			}
 		}
 		log.WithFields(log.Fields{
-			"rid":         rid,
-			"options":     sigreq.Options,
-			"mode":        sigresps[i].Mode,
-			"ref":         sigresps[i].Ref,
-			"type":        sigresps[i].Type,
-			"signer_id":   sigresps[i].SignerID,
-			"input_hash":  inputHash,
-			"output_hash": outputHash,
-			"user_id":     userid,
-			"t":           int32(time.Since(starttime) / time.Millisecond), //  request processing time in ms
+			"rid":           rid,
+			"options":       sigreq.Options,
+			"mode":          sigresps[i].Mode,
+			"ref":           sigresps[i].Ref,
+			"type":          sigresps[i].Type,
+			"signer_id":     sigresps[i].SignerID,
+			"input_hash":    inputHash,
+			"input_hashes":  inputHashes,
+			"output_hash":   outputHash,
+			"output_hashes": outputHashes,
+			"user_id":       userid,
+			"t":             int32(time.Since(starttime) / time.Millisecond), //  request processing time in ms
 		}).Info("signing operation succeeded")
 	}
 	respdata, err := json.Marshal(sigresps)