From bb85c068096ea2d4795432188f6d05b453c6ee7e Mon Sep 17 00:00:00 2001 From: Christopher Maddalena Date: Tue, 24 Sep 2024 13:23:11 -0700 Subject: [PATCH 1/5] Fixed domains marked as expired not returning to available if renewed --- ghostwriter/shepherd/forms.py | 2 +- ghostwriter/shepherd/tasks.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/ghostwriter/shepherd/forms.py b/ghostwriter/shepherd/forms.py index d8444ad00..41b9689eb 100644 --- a/ghostwriter/shepherd/forms.py +++ b/ghostwriter/shepherd/forms.py @@ -183,7 +183,7 @@ def __init__(self, *args, **kwargs): for field in self.fields: self.fields[field].widget.attrs["autocomplete"] = "off" self.fields["name"].widget.attrs["placeholder"] = "ghostwriter.wiki" - self.fields["registrar"].widget.attrs["placeholder"] = "NameCheap" + self.fields["registrar"].widget.attrs["placeholder"] = "Namecheap" self.fields["domain_status"].empty_label = "-- Select Status --" self.fields["whois_status"].empty_label = "-- Select Status --" self.fields["health_status"].empty_label = "-- Select Status --" diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 1e994fef8..36bacdb63 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -760,6 +760,8 @@ def fetch_namecheap_domains(): domain_queryset = Domain.objects.filter(registrar="Namecheap") expired_status = DomainStatus.objects.get(domain_status="Expired") burned_status = DomainStatus.objects.get(domain_status="Burned") + unavailable_status = DomainStatus.objects.get(domain_status="Unavailable") + available_status = DomainStatus.objects.get(domain_status="Available") health_burned_status = HealthStatus.objects.get(health_status="Burned") for domain in domain_queryset: # Check if a domain in the library is _not_ in the Namecheap response @@ -799,6 +801,17 @@ def fetch_namecheap_domains(): domain=domain, note="Automatically set to Expired because the domain did not appear in Namecheap during a sync.", ) + # Catch domains that were marked as expired but are now back in the Namecheap data + else: + if domain.expired: + logger.info("Domain %s is marked as expired but is now back in the Namecheap data", domain.name) + domain_changes["updates"][domain.id] = {} + domain_changes["updates"][domain.id]["domain"] = domain.name + domain_changes["updates"][domain.id]["change"] = "renewed" + domain.expired = False + domain.domain_status = available_status + domain.save() + # Now, loop over every domain returned by Namecheap for domain in domains_list: logger.info("Domain %s is now being processed", domain["Name"]) @@ -840,8 +853,12 @@ def fetch_namecheap_domains(): ] = "

Namecheap has locked the domain. This is usually the result of a legal complaint related to phishing/malicious activities.

" # Set AutoRenew status + # Ignore Namecheap's `AutoRenew` value if the domain is expired (both can be true) if domain["AutoRenew"] == "false" or domain["IsExpired"] == "true": entry["auto_renew"] = False + # Ensure the domain's auto-renew status in the database matches Namecheap + elif domain["AutoRenew"] == "true": + entry["auto_renew"] = True # Convert Namecheap dates to Django entry["creation"] = datetime.strptime(domain["Created"], "%m/%d/%Y").strftime("%Y-%m-%d") From 12c789dd7ebf021df12a625b15b13c052cece78f Mon Sep 17 00:00:00 2001 From: Christopher Maddalena Date: Tue, 24 Sep 2024 13:29:47 -0700 Subject: [PATCH 2/5] Added check to only change status if it was set to Expired --- ghostwriter/shepherd/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 36bacdb63..922683486 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -760,7 +760,6 @@ def fetch_namecheap_domains(): domain_queryset = Domain.objects.filter(registrar="Namecheap") expired_status = DomainStatus.objects.get(domain_status="Expired") burned_status = DomainStatus.objects.get(domain_status="Burned") - unavailable_status = DomainStatus.objects.get(domain_status="Unavailable") available_status = DomainStatus.objects.get(domain_status="Available") health_burned_status = HealthStatus.objects.get(health_status="Burned") for domain in domain_queryset: @@ -809,7 +808,8 @@ def fetch_namecheap_domains(): domain_changes["updates"][domain.id]["domain"] = domain.name domain_changes["updates"][domain.id]["change"] = "renewed" domain.expired = False - domain.domain_status = available_status + if domain.domain_status == expired_status: + domain.domain_status = available_status domain.save() # Now, loop over every domain returned by Namecheap From a9c23f7eeefca008bc99a9cc39426d036ab05efe Mon Sep 17 00:00:00 2001 From: Christopher Maddalena Date: Wed, 25 Sep 2024 10:42:49 -0700 Subject: [PATCH 3/5] Added `replace_blanks` filter for templates --- ghostwriter/modules/reportwriter/__init__.py | 2 ++ .../modules/reportwriter/jinja_funcs.py | 23 +++++++++++++++++++ ghostwriter/reporting/tests/test_views.py | 14 +++++++++++ 3 files changed, 39 insertions(+) diff --git a/ghostwriter/modules/reportwriter/__init__.py b/ghostwriter/modules/reportwriter/__init__.py index 1bf08999b..29f22c14b 100644 --- a/ghostwriter/modules/reportwriter/__init__.py +++ b/ghostwriter/modules/reportwriter/__init__.py @@ -41,6 +41,7 @@ def __iter__(self): def __bool__(self): self._record() return super().__bool__() + undefined = RecordUndefined else: undefined = jinja2.make_logging_undefined(logger=logger, base=jinja2.Undefined) @@ -55,6 +56,7 @@ def __bool__(self): env.filters["get_item"] = jinja_funcs.get_item env.filters["regex_search"] = jinja_funcs.regex_search env.filters["filter_tags"] = jinja_funcs.filter_tags + env.filters["replace_blanks"] = jinja_funcs.replace_blanks if debug: return env, undefined_vars diff --git a/ghostwriter/modules/reportwriter/jinja_funcs.py b/ghostwriter/modules/reportwriter/jinja_funcs.py index a52dfa4a5..ec51e93a5 100644 --- a/ghostwriter/modules/reportwriter/jinja_funcs.py +++ b/ghostwriter/modules/reportwriter/jinja_funcs.py @@ -282,3 +282,26 @@ def mk_evidence(context: jinja2.runtime.Context, evidence_name: str) -> Markup: def raw_mk_evidence(evidence_id) -> Markup: return Markup('') + + +def replace_blanks(list_of_dicts, placeholder=""): + """ + Replace blank strings in a dictionary with a placeholder string. + + **Parameters** + + ``dict`` + Dictionary to replace blanks in + """ + + try: + for d in list_of_dicts: + for key, value in d.items(): + if value is None: + d[key] = placeholder + except (AttributeError, TypeError) as e: + logger.exception("Error parsing ``list_of_dicts`` as a list of dictionaries: %s", list_of_dicts) + raise InvalidFilterValue( + "Invalid list of dictionaries passed into `replace_blanks()` filter; must be a list of dictionaries" + ) from e + return list_of_dicts diff --git a/ghostwriter/reporting/tests/test_views.py b/ghostwriter/reporting/tests/test_views.py index 25204fa28..84333a407 100644 --- a/ghostwriter/reporting/tests/test_views.py +++ b/ghostwriter/reporting/tests/test_views.py @@ -53,6 +53,7 @@ get_item, regex_search, strip_html, + replace_blanks, ) from ghostwriter.reporting.templatetags import report_tags @@ -2508,6 +2509,19 @@ def test_filter_tags_with_invalid_dict(self): with self.assertRaises(InvalidFilterValue): filter_tags(findings, ["xss", "T1659"]) + def test_replace_blanks(self): + example = [ + {"example": "This is a test"}, + {"example": None}, + {"example": "This is another test"}, + ] + res = replace_blanks(example, "BLANK") + self.assertEqual( + res, [{"example": "This is a test"}, {"example": "BLANK"}, {"example": "This is another test"}] + ) + with self.assertRaises(InvalidFilterValue): + replace_blanks("Not a list", "BLANK") + class LocalFindingNoteUpdateTests(TestCase): """Collection of tests for :view:`reporting.LocalFindingNoteUpdate`.""" From 8dfc2676234c107de700eeda1090dd2e65758398 Mon Sep 17 00:00:00 2001 From: Christopher Maddalena Date: Wed, 25 Sep 2024 12:50:02 -0700 Subject: [PATCH 4/5] Changed serializer to replace null values with blank strings --- ghostwriter/modules/custom_serializers.py | 31 +++++++++++++++-------- requirements/base.txt | 6 ++--- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/ghostwriter/modules/custom_serializers.py b/ghostwriter/modules/custom_serializers.py index 5d46253cc..0d564cd37 100644 --- a/ghostwriter/modules/custom_serializers.py +++ b/ghostwriter/modules/custom_serializers.py @@ -77,6 +77,20 @@ def __init__(self, *args, exclude=None, **kwargs): self.fields.pop(field) super().__init__(*args, **kwargs) + def to_representation(self, instance): + """ + Override the default method to ensure empty strings are returned for null values. The null values will + cause Jinja2 rendering errors with filters and expressions like `sort()`. + """ + data = super().to_representation(instance) + for key, value in data.items(): + try: + if not value: + data[key] = "" + except KeyError: + pass + return data + class OperatorNameField(RelatedField): """Customize the string representation of a :model:`users.User` entry.""" @@ -113,7 +127,7 @@ class ExtraFieldsSerField(serializers.Field): def __init__(self, model_name, **kwargs): self.model_name = model_name self.root_ser = None - kwargs['read_only'] = True + kwargs["read_only"] = True super().__init__(**kwargs) def bind(self, field_name, parent): @@ -130,7 +144,9 @@ def to_representation(self, value): if not hasattr(self.root_ser, "_extra_fields_specs") or self.root_ser._extra_fields_specs is None: self.root_ser._extra_fields_specs = {} if self.model_name not in self.root_ser._extra_fields_specs: - self.root_ser._extra_fields_specs[self.model_name] = ExtraFieldSpec.objects.filter(target_model=self.model_name) + self.root_ser._extra_fields_specs[self.model_name] = ExtraFieldSpec.objects.filter( + target_model=self.model_name + ) # Populate output for field in self.root_ser._extra_fields_specs[self.model_name]: @@ -514,10 +530,7 @@ class DomainHistorySerializer(CustomModelSerializer): exclude=["id", "project", "domain"], ) - extra_fields = ExtraFieldsSerField( - Domain._meta.label, - source="domain.extra_fields" - ) + extra_fields = ExtraFieldsSerField(Domain._meta.label, source="domain.extra_fields") class Meta: model = History @@ -567,10 +580,7 @@ class ServerHistorySerializer(CustomModelSerializer): exclude=["id", "project", "static_server", "transient_server"], ) - extra_fields = ExtraFieldsSerField( - StaticServer._meta.label, - source="server.extra_fields" - ) + extra_fields = ExtraFieldsSerField(StaticServer._meta.label, source="server.extra_fields") class Meta: model = ServerHistory @@ -756,6 +766,7 @@ class Meta: class FullProjectSerializer(serializers.Serializer): """Serialize :model:`rolodex:Project` and related entries.""" + project = ProjectSerializer(source="*") client = ClientSerializer() contacts = ProjectContactSerializer(source="projectcontact_set", many=True, exclude=["id", "project"]) diff --git a/requirements/base.txt b/requirements/base.txt index e568668b0..8a03adf56 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -34,12 +34,12 @@ djangorestframework-api-key==2.2.0 django-tinymce==3.4.0 # Deprecated, but kept here for legacy migrations docutils==0.18.1 docxtpl==0.12.0 -dnspython==2.2.0 -jinja2==3.1.3 +dnspython==2.6.1 +jinja2==3.1.4 python-docx==0.8.11 python-nmap==0.7.1 python-pptx==0.6.21 -requests==2.31.0 +requests==2.32.0 django-timezone-field==4.2.3 pyjwt==2.7.0 psutil==5.9.4 From 4ee66a114f83746adc97897ffcd18eb1c1b43641 Mon Sep 17 00:00:00 2001 From: Christopher Maddalena Date: Wed, 25 Sep 2024 13:54:28 -0700 Subject: [PATCH 5/5] Updated for staging --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a67e8836d..ec0903a3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 # CHANGELOG +## [4.3.1] – 25 Sep 2024 + +### Added + +* Added a `replace_blanks` filter to the report template engine to replace blank values in a dictionary with a specified string + * This filter is useful when sorting a list of dictionaries with an attribute that may have a blank value +* Added an option in the change search in the findings library to search findings attached to reports (Closes #400) + * Instead of matches from the library, the search will return results for findings attached to reports to which the user has access + +### Changed + +* Changed the serializer for report context to replace null values with a blank string (`""`) to help prevent errors when generating reports + * **Note:** This change may affect templates that rely on null values to trigger conditional logic, but most conditional statements should not be affected + * **Example:** The condition `{% if not X %}` will still evaluate to `True` if `X` is `None` or `""` +* Changed the report form to allow users with the `admin` or `manager` roles to change the report's project (Closes #368) + * This change allows a report to be moved from one project to another (e.g., you make a copy for a follow-up assessment) + * This feature is only available to users with the `admin` or `manager` roles to prevent accidental data leaks + +### Fixed + +* Fixed an edge case with the Namecheap sync task that could lead to a domain remaining marked as expired after re-purchasing it or renewing it during the grace period + ## [4.3.0] – 23 Sep 2024 ### Added