Remove Vigilion code and references

Use Carrierwave for file uploads
Update to use DBT-Scanner via URL with credentials
Add specs for new functionality and update existing specs where appropriate

.env.example

@@ -3,17 +3,22 @@ DEVISE_SECRET_KEY=DEVISE_SECRET_KEY
 SENTRY_DSN=SENTRY_DSN
 AUTHY_API_KEY=AUTHY_API_KEY
 AUTHY_API_URL=https://api.authy.com
-VIGILION_ACCESS_KEY_ID=XXXXXXXXXXXXXXXXXX
-VIGILION_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXX
 DISABLE_VIRUS_SCANNER=false
+ENABLE_VIRUS_SCANNER_BUCKETS=false
+VIRUS_SCANNER_URL=http://localhost:80
+VIRUS_SCANNER_USERNAME=app1
+VIRUS_SCANNER_PASSWORD=letmein
 PROFILE_MODE=false
 LOG_LEVEL=info
 ASSET_HOST=http://localhost:3000/
 MAILER_HOST=localhost
-AWS_ACCESS_KEY_ID=xxx
-AWS_SECRET_ACCESS_KEY=xxx
+AWS_TMP_BUCKET_ACCESS_KEY_ID=xxx
+AWS_TMP_BUCKET_SECRET_ACCESS_KEY=xxx
+AWS_PERMANENT_BUCKET_ACCESS_KEY_ID=xxx
+AWS_PERMANENT_BUCKET_SECRET_ACCESS_KEY=xxx
 AWS_REGION=xxx
-AWS_S3_BUCKET_NAME=xxx
+AWS_S3_TMP_BUCKET=xxx
+AWS_S3_PERMANENT_BUCKET=xxx
 DISPLAY_SOCIAL_MOBILITY_AWARD=true
 PUSHER_SOCKET_HOST=localhost
 PUSHER_WS_PORT=8080

.env.test

@@ -3,9 +3,6 @@ DEVISE_SECRET_KEY=DEVISE_SECRET_KEY
 SENTRY_DSN=SENTRY_DSN
 AUTHY_API_KEY=AUTHY_API_KEY
 AUTHY_API_URL=https://api.authy.com
-VIGILION_ACCESS_KEY_ID=XXXXXXXXXXXXXXXXXX
-VIGILION_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXX
-DISABLE_VIRUS_SCANNER=true
 PROFILE_MODE=false
 LOG_LEVEL=info
 ASSET_HOST=

.snyk

@@ -2,10 +2,6 @@
 version: v1.7.0
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
 ignore:
-  SNYK-RUBY-FARADAYMIDDLEWARE-20334:
-    - '* > [email protected]':
-        reason: None given
-        expires: '2017-06-09T12:30:08.169Z'
   SNYK-RUBY-NOKOGIRI-20299:
     - '*':
         reason: an application using Nokogiri needs to be opt into the DTDLOAD option and opt out of the NONET option in order to be vulnerable

Gemfile

@@ -91,8 +91,7 @@ gem "nokogiri"
 # Uploads
 gem "carrierwave", "~> 3.0"
 gem "fog-aws"
-gem "vigilion", "~> 1.0.4"
-gem "vigilion-rails", "~> 2.2.0"
+gem "aws-sdk-s3", "~> 1"
 
 # Background jobs
 gem "sidekiq", "~> 6.5.10"

Gemfile.lock

@@ -83,6 +83,22 @@ GEM
       httpclient (>= 2.5.3.3)
     autoprefixer-rails (10.4.13.0)
       execjs (~> 2)
+    aws-eventstream (1.3.0)
+    aws-partitions (1.983.0)
+    aws-sdk-core (3.209.1)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.9)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.94.0)
+      aws-sdk-core (~> 3, >= 3.207.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-s3 (1.167.0)
+      aws-sdk-core (~> 3, >= 3.207.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.10.0)
+      aws-eventstream (~> 1, >= 1.0.2)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
@@ -205,33 +221,6 @@ GEM
     factory_bot_rails (6.2.0)
       factory_bot (~> 6.2.0)
       railties (>= 5.0.0)
-    faraday (1.10.3)
-      faraday-em_http (~> 1.0)
-      faraday-em_synchrony (~> 1.0)
-      faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0)
-      faraday-multipart (~> 1.0)
-      faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.0)
-      faraday-patron (~> 1.0)
-      faraday-rack (~> 1.0)
-      faraday-retry (~> 1.0)
-      ruby2_keywords (>= 0.0.4)
-    faraday-detailed_logger (2.5.0)
-      faraday (>= 0.16, < 3)
-    faraday-em_http (1.0.0)
-    faraday-em_synchrony (1.0.0)
-    faraday-excon (1.1.0)
-    faraday-httpclient (1.0.1)
-    faraday-multipart (1.0.4)
-      multipart-post (~> 2)
-    faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.2.0)
-    faraday-patron (1.0.0)
-    faraday-rack (1.0.0)
-    faraday-retry (1.0.3)
-    faraday_middleware (1.2.0)
-      faraday (~> 1.0)
     ffi (1.15.5)
     fog-aws (3.19.0)
       fog-core (~> 2.1)
@@ -281,6 +270,7 @@ GEM
       ruby-vips (>= 2.0.17, < 3)
     jbuilder (2.10.2)
       activesupport (>= 5.0.0)
+    jmespath (1.6.2)
     jquery-rails (4.4.0)
       rails-dom-testing (>= 1, < 3)
       railties (>= 4.2.0)
@@ -349,7 +339,6 @@ GEM
     money (6.16.0)
       i18n (>= 0.6.4, <= 2)
     multi_json (1.15.0)
-    multipart-post (2.3.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
     net-imap (0.4.10)
@@ -683,14 +672,6 @@ GEM
       activesupport (>= 5.2.0, < 8.0)
       concurrent-ruby (~> 1.0)
       method_source (~> 1.0)
-    vigilion (1.0.4)
-      addressable (~> 2)
-      faraday
-      faraday-detailed_logger
-      faraday_middleware
-    vigilion-rails (2.2.0)
-      rails (>= 6.0.3.7)
-      vigilion (~> 1.0.4)
     virtus (2.0.0)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
@@ -730,6 +711,7 @@ PLATFORMS
 DEPENDENCIES
   active_hash
   amoeba (= 3.0.0)
+  aws-sdk-s3 (~> 1)
   binding_of_caller
   bootscale
   bootstrap-sass (~> 3.4)
@@ -826,8 +808,6 @@ DEPENDENCIES
   timecop
   turnip (~> 4.2.0)
   uglifier (>= 2.7.2)
-  vigilion (~> 1.0.4)
-  vigilion-rails (~> 2.2.0)
   virtus
   webmock (= 3.18.1)
   webpacker (= 6.0.0.rc.6)

Procfile

@@ -1,2 +1,2 @@
 web: bundle exec rake cf:run_migrations db:migrate && bundle exec puma -C config/puma.rb
-worker: bundle exec sidekiq -L ./log/worker.log -C ./config/sidekiq.yml
+worker: bundle exec sidekiq -C ./config/sidekiq.yml

Procfile.dev

@@ -1,2 +1,2 @@
 web: bundle exec rake cf:run_migrations db:migrate && bundle exec puma -C config/puma.rb
-worker: bundle exec sidekiq -L ./log/worker.log -C ./config/sidekiq.yml
+worker: bundle exec sidekiq -C ./config/sidekiq.yml

README.md

@@ -62,9 +62,18 @@ If you need to test collaborators editing the application at the same time, inst
 
 ### Installing Malware Scanning
 
-Files are uploaded to S3 and then scanned with ClamAV via the Vigilion service.
+Files are uploaded to S3 and then scanned with ClamAV via the DBT scanner service.
+
+If you need to test malware scanning locally, run the [DBT scanner](https://github.com/uktrade/dit-clamav-rest) via docker-compose.
+
+You will also need to set the following environment variables in the `.env` file:
+
+```
+VIRUS_SCANNER_URL=http://localhost:80
+VIRUS_SCANNER_USERNAME=app1
+VIRUS_SCANNER_PASSWORD=letmein
+```
 
-If you need to test malware scanning locally, install [Vigilion](https://github.com/bitzesty/vigilion-scanner) and set the `VIGILION_ACCESS_KEY_ID` and `VIGILION_SECRET_ACCESS_KEY` and `DISABLE_VIRUS_SCANNER` to `false` in the `.env` file.
 
 ### Running the tests
 

app/jobs/file_scan_job.rb

@@ -0,0 +1,66 @@
+class FileScanJob < ApplicationJob
+  queue_as :default
+
+  def perform(key, class_name, record_id, attribute_name)
+    record = class_name.constantize.find(record_id)
+    file = record.send(attribute_name)
+
+    return if file.blank?
+
+    begin
+      file_to_scan = get_file_to_scan(file)
+      scan_result = VirusScanner.scan_file(file_to_scan)
+      status = scan_result[:malware] ? "infected" : "clean"
+      record.send(:"on_scan_#{attribute_name}", status: status)
+    rescue VirusScanner::AuthenticationError => e
+      handle_authentication_error(record, attribute_name, e)
+    rescue VirusScanner::FileTooLargeError => e
+      handle_file_too_large_error(record, attribute_name, e)
+    rescue VirusScanner::ScanError => e
+      handle_scan_error(record, attribute_name, e)
+    ensure
+      file_to_scan.close if file_to_scan.respond_to?(:close)
+    end
+  end
+
+  private
+
+  def get_file_to_scan(file)
+    if file.is_a?(String)
+      File.open(file, "rb")
+    elsif file.respond_to?(:read)
+      file
+    elsif file.is_a?(CarrierWave::SanitizedFile)
+      File.open(file.file, "rb")
+    elsif file.respond_to?(:file)
+      if file.file.is_a?(CarrierWave::SanitizedFile)
+        File.open(file.file.file, "rb")
+      elsif file.file.respond_to?(:path)
+        File.open(file.file.path, "rb")
+      elsif file.file.respond_to?(:read)
+        file.file
+      else
+        raise ArgumentError, "Don't know how to handle #{file.file.class}"
+      end
+    elsif file.respond_to?(:path)
+      File.open(file.path, "rb")
+    else
+      raise ArgumentError, "Don't know how to handle #{file.class}"
+    end
+  end
+
+  def handle_authentication_error(record, attribute_name, error)
+    Rails.logger.error("VirusScanner Authentication Error: #{error.message}")
+    record.send(:"on_scan_#{attribute_name}", status: :error)
+  end
+
+  def handle_file_too_large_error(record, attribute_name, error)
+    Rails.logger.warn("File too large for virus scanning: #{error.message}")
+    record.send(:"on_scan_#{attribute_name}", status: :error)
+  end
+
+  def handle_scan_error(record, attribute_name, error)
+    Rails.logger.error("VirusScanner Error: #{error.message}")
+    record.send(:"on_scan_#{attribute_name}", status: :error)
+  end
+end