Check links and redirects against a domain list

Currently this only includes a very small list of suspicious domains
which we want to check as a sort of one-off.

We could consider using a third party list of domains (or IP ranges)
instead, if this is considered to be valuable.

app/lib/link_checker/uri_checker/http_checker.rb

@@ -35,6 +35,12 @@ def initialize(options = {})
     end
   end
 
+  class SuspiciousDomain < LinkChecker::UriChecker::Warning
+    def initialize(options = {})
+      super(summary: :suspicious_destination, message: :website_on_list_of_suspicious_domains, **options)
+    end
+  end
+
   class SlowResponse < LinkChecker::UriChecker::Warning
     def initialize(options = {})
       super(summary: :slow_page, message: :page_is_slow, suggested_fix: :contact_site_administrator, **options)
@@ -104,6 +110,7 @@ def call
       check_redirects
       check_credentials_in_uri
       check_top_level_domain
+      check_suspicious_domains
 
       check_request
       return report if report.has_errors?
@@ -119,6 +126,7 @@ def call
     attr_reader :response
 
     INVALID_TOP_LEVEL_DOMAINS = %w[xxx adult dating porn sex sexy singles].freeze
+    SUSPICIOUS_DOMAINS = Rails.application.config_for(:domains).suspicious_domains.freeze
     REDIRECT_STATUS_CODES = [301, 302, 303, 307, 308].freeze
     REDIRECT_LIMIT = 8
     REDIRECT_LOOP_LIMIT = 5
@@ -145,6 +153,12 @@ def check_top_level_domain
       end
     end
 
+    def check_suspicious_domains
+      if SUSPICIOUS_DOMAINS.any? { |d| uri.host.ends_with? d }
+        add_problem(SuspiciousDomain.new(from_redirect: from_redirect?))
+      end
+    end
+
     def check_request
       start_time = Time.zone.now
       @response = make_request(:get)

app/lib/link_checker/uri_checker/problem.rb

@@ -56,6 +56,7 @@ def get_string(symbol)
       TooManyRedirectsSlowly
       CredentialsInUri
       SuspiciousTld
+      SuspiciousDomain
       SlowResponse
       PageWithRating
       PageContainsThreat

config/domains.yml

@@ -0,0 +1,19 @@
+default: &default
+  suspicious_domains:
+    # NOTE: These domains are considered suspicious by GDS corporate IT
+    #       making requests from the corporate network may raise flags.
+    - nostringsng.com
+    - www.becauseiamagirl.org
+    - www.bilebrizoua.ci
+    # NOTE: This is not a comprehensive list of all suspicious domains on the internet.
+
+test:
+  <<: *default
+  suspicious_domains:
+    - malicious.example.com
+
+development:
+  <<: *default
+
+production:
+  <<: *default

config/locales/en.yml

@@ -49,6 +49,10 @@ en:
     singular: This link is hosted on a website meant for adult content.
     redirect: This redirects to websites meant for adult content.
 
+  website_on_list_of_suspicious_domains:
+    singular: This link is hosted on a domain which is on our list of suspicious domains
+    redirect: This redirects to a website which is on our list of suspicious domains
+
   slow_page: Slow page
   page_is_slow:
     singular: This page is slow loading and may frustrate users.

spec/lib/link_checker_spec.rb

@@ -96,6 +96,14 @@
       include_examples "has no errors"
     end
 
+    context "domain is risky" do
+      let(:uri) { "https://malicious.example.com" }
+      before { stub_request(:get, uri).to_return(status: 200) }
+      include_examples "has a problem summary", "Suspicious Destination"
+      include_examples "has warnings"
+      include_examples "has no errors"
+    end
+
     context "there are credentials in the URI" do
       let(:uri) { "https://username:[email protected]/ok" }
       include_examples "has a problem summary", "Login details in URL"