From 8734eb640a7416e549244c5ee012553ba9ced2cf Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Tue, 25 Jul 2023 17:49:11 -0700 Subject: [PATCH 1/8] fix: hook failure due to urllib3 update --- .../request/patch/hook_urllib3_parse_url.py | 65 +++++++++---------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/pocsuite3/lib/request/patch/hook_urllib3_parse_url.py b/pocsuite3/lib/request/patch/hook_urllib3_parse_url.py index ad35c71e..f3d33861 100644 --- a/pocsuite3/lib/request/patch/hook_urllib3_parse_url.py +++ b/pocsuite3/lib/request/patch/hook_urllib3_parse_url.py @@ -115,39 +115,6 @@ def __str__(self): return self.url -def split_first(s, delims): - """ - Given a string and an iterable of delimiters, split on the first found - delimiter. Return two split parts and the matched delimiter. - - If not found, then the first part is the full input string. - - Example:: - - >>> split_first('foo/bar?baz', '?/=') - ('foo', 'bar?baz', '/') - >>> split_first('foo/bar?baz', '123') - ('foo/bar?baz', '', None) - - Scales linearly with number of delims. Not ideal for large number of delims. - """ - min_idx = None - min_delim = None - for d in delims: - idx = s.find(d) - if idx < 0: - continue - - if min_idx is None or idx < min_idx: - min_idx = idx - min_delim = d - - if min_idx is None or min_idx < 0: - return s, '', None - - return s[:min_idx], s[min_idx + 1:], min_delim - - def patched_parse_url(url): """ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is @@ -170,6 +137,38 @@ def patched_parse_url(url): # Additionally, this implementations does silly things to be optimal # on CPython. + def split_first(s, delims): + """ + Given a string and an iterable of delimiters, split on the first found + delimiter. Return two split parts and the matched delimiter. + + If not found, then the first part is the full input string. + + Example:: + + >>> split_first('foo/bar?baz', '?/=') + ('foo', 'bar?baz', '/') + >>> split_first('foo/bar?baz', '123') + ('foo/bar?baz', '', None) + + Scales linearly with number of delims. Not ideal for large number of delims. + """ + min_idx = None + min_delim = None + for d in delims: + idx = s.find(d) + if idx < 0: + continue + + if min_idx is None or idx < min_idx: + min_idx = idx + min_delim = d + + if min_idx is None or min_idx < 0: + return s, '', None + + return s[:min_idx], s[min_idx + 1:], min_delim + if not url: # Empty return Url() From 4eb6314362cf3c6593753acc014315f56685ac38 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 26 Jul 2023 11:49:29 -0700 Subject: [PATCH 2/8] fix: optimize DSL expression execution --- pocsuite3/lib/request/patch/add_httpraw.py | 2 +- .../protocols/common/expressions/__init__.py | 93 +++++++++++++++++-- .../protocols/common/expressions/safe_eval.py | 4 +- .../yaml/nuclei/protocols/http/__init__.py | 28 +++--- .../yaml/nuclei/protocols/network/__init__.py | 26 +++--- pocsuite3/modules/interactsh/__init__.py | 5 +- 6 files changed, 118 insertions(+), 40 deletions(-) diff --git a/pocsuite3/lib/request/patch/add_httpraw.py b/pocsuite3/lib/request/patch/add_httpraw.py index de77e156..6ab0c2cf 100644 --- a/pocsuite3/lib/request/patch/add_httpraw.py +++ b/pocsuite3/lib/request/patch/add_httpraw.py @@ -49,7 +49,7 @@ def httpraw(raw: str, ssl: bool = False, **kwargs): raise Exception tmp_headers = raws[1:index - 1] tmp_headers = extract_dict('\n'.join(tmp_headers), '\n', ": ") - postData = raws[index] + postData = '\n'.join(raws[index:]) try: json.loads(postData) _json = postData diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py index 27df717d..2c535172 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py @@ -5,17 +5,19 @@ import hashlib import hmac as py_hmac import html +import inspect import random import re import string import time import urllib.parse import zlib as py_built_in_zlib -from typing import Union +from functools import wraps +from typing import get_type_hints, Union +import chardet import mmh3 as py_mmh3 from pkg_resources import parse_version - from pocsuite3.lib.core.log import LOGGER as logger from pocsuite3.lib.yaml.nuclei.protocols.common.expressions.safe_eval import safe_eval @@ -31,6 +33,52 @@ class Marker: ParenthesisClose = "}}" +def auto_convert_types(func): + @wraps(func) + def check_and_convert_args(*args, **kwargs): + # Get the function's parameter names and types + signature = inspect.signature(func) + parameter_types = get_type_hints(func) + + # Convert args to a list so we can modify its elements + args_list = list(args) + + # Check and convert positional arguments + for i, (arg_name, arg_value) in enumerate(zip(signature.parameters.keys(), args)): + arg_type = parameter_types.get(arg_name) + if arg_type and not isinstance(arg_value, arg_type): + try: + if arg_type is str and isinstance(arg_value, bytes): + try: + encoding = chardet.detect(arg_value)['encoding'] or 'utf-8' + args_list[i] = arg_value.decode(encoding) + except Exception: + args_list[i] = str(arg_value) + elif arg_type is bytes and isinstance(arg_value, str): + args_list[i] = arg_value.encode('utf-8') + except ValueError: + pass + # Check and convert keyword arguments + for arg_name, arg_value in kwargs.items(): + arg_type = parameter_types.get(arg_name) + if arg_type and not isinstance(arg_value, arg_type): + try: + if arg_type is str and isinstance(arg_value, bytes): + try: + encoding = chardet.detect(arg_value)['encoding'] or 'utf-8' + kwargs[arg_name] = arg_value.decode(encoding) + except Exception: + kwargs[arg_name] = str(arg_value) + elif arg_type is bytes and isinstance(arg_value, str): + kwargs[arg_name] = arg_value.encode('utf-8') + except ValueError: + pass + # Call the original function with the potentially converted arguments + return func(*args_list, **kwargs) + + return check_and_convert_args + + def aes_gcm(key: Union[bytes, str], plaintext: Union[bytes, str]) -> bytes: """ AES GCM encrypts a string with key @@ -77,7 +125,8 @@ def base64_py(src: Union[bytes, str]) -> str: return base64(src) -def concat(*arguments) -> str: +@auto_convert_types +def concat(*arguments: str) -> str: """ Concatenates the given number of arguments to form a string @@ -88,6 +137,7 @@ def concat(*arguments) -> str: return ''.join(map(str, arguments)) +@auto_convert_types def compare_versions(version_to_check: str, *constraints: str) -> bool: """ Compares the first version argument with the provided constraints @@ -112,6 +162,7 @@ def compare_versions(version_to_check: str, *constraints: str) -> bool: return True +@auto_convert_types def contains(inp: str, substring: str) -> bool: """ Verifies if a string contains a substring @@ -123,6 +174,7 @@ def contains(inp: str, substring: str) -> bool: return substring in inp +@auto_convert_types def contains_all(inp: str, *substrings: str) -> bool: """ Verify if any input contains all the substrings @@ -134,6 +186,7 @@ def contains_all(inp: str, *substrings: str) -> bool: return all(map(lambda s: s in inp, substrings)) +@auto_convert_types def contains_any(inp: str, *substrings: str) -> bool: """ Verifies if an input contains any of substrings @@ -218,6 +271,7 @@ def gzip(inp: Union[str, bytes]) -> bytes: return py_built_in_gzip.compress(inp) +@auto_convert_types def gzip_decode(inp: bytes) -> bytes: """ Decompresses the input using GZip @@ -242,6 +296,7 @@ def zlib(inp: Union[str, bytes]) -> bytes: return py_built_in_zlib.compress(inp) +@auto_convert_types def zlib_decode(inp: bytes) -> bytes: """ Decompresses the input using Zlib @@ -253,6 +308,7 @@ def zlib_decode(inp: bytes) -> bytes: return py_built_in_zlib.decompress(inp) +@auto_convert_types def hex_decode(inp: str) -> bytes: """ Hex decodes the given input @@ -277,6 +333,7 @@ def hex_encode(inp: Union[str, bytes]) -> str: return binascii.hexlify(inp).decode('utf-8') +@auto_convert_types def html_escape(inp: str) -> str: """ HTML escapes the given input @@ -288,6 +345,7 @@ def html_escape(inp: str) -> str: return html.escape(inp) +@auto_convert_types def html_unescape(inp: str) -> str: """ HTML un-escapes the given input @@ -314,7 +372,8 @@ def md5(inp: Union[str, bytes]) -> str: return m.hexdigest() -def mmh3(inp: Union[str, bytes]) -> int: +@auto_convert_types +def mmh3(inp: str) -> int: """ Calculates the MMH3 (MurmurHash3) hash of an input @@ -406,7 +465,8 @@ def rand_text_numeric(length: int, optional_bad_numbers: str = '') -> str: return ''.join(random.choice(charset) for _ in range(length)) -def regex(pattern, inp) -> bool: +@auto_convert_types +def regex(pattern: str, inp: str) -> bool: """ Tests the given regular expression against the input string @@ -414,9 +474,10 @@ def regex(pattern, inp) -> bool: Input: regex("H([a-z]+)o", "Hello") Output: True """ - return re.findall(pattern, inp) != [] + return list(filter(lambda item: item.strip() != "", re.findall(pattern, inp, re.IGNORECASE))) != [] +@auto_convert_types def remove_bad_chars(inp: str, cutset: str) -> str: """ Removes the desired characters from the input @@ -428,6 +489,7 @@ def remove_bad_chars(inp: str, cutset: str) -> str: return ''.join(i if i not in cutset else '' for i in inp) +@auto_convert_types def repeat(inp: str, count: int) -> str: """ Repeats the input string the given amount of times @@ -439,6 +501,7 @@ def repeat(inp: str, count: int) -> str: return inp * count +@auto_convert_types def replace(inp: str, old: str, new: str) -> str: """ Replaces a given substring in the given input @@ -450,6 +513,7 @@ def replace(inp: str, old: str, new: str) -> str: return inp.replace(old, new) +@auto_convert_types def replace_regex(source: str, pattern: str, replacement: str) -> str: """ Replaces substrings matching the given regular expression in the input @@ -461,6 +525,7 @@ def replace_regex(source: str, pattern: str, replacement: str) -> str: return re.sub(pattern, replacement, source) +@auto_convert_types def reverse(inp: str) -> str: """ Reverses the given input @@ -504,6 +569,7 @@ def sha256(inp: Union[bytes, str]) -> str: return s.hexdigest() +@auto_convert_types def to_lower(inp: str) -> str: """ Transforms the input into lowercase characters @@ -515,6 +581,7 @@ def to_lower(inp: str) -> str: return inp.lower() +@auto_convert_types def to_upper(inp: str) -> str: """ Transforms the input into uppercase characters @@ -526,6 +593,7 @@ def to_upper(inp: str) -> str: return inp.upper() +@auto_convert_types def trim(inp: str, cutset: str) -> str: """ Returns a slice of the input with all leading and trailing Unicode code points contained in cutset removed @@ -537,6 +605,7 @@ def trim(inp: str, cutset: str) -> str: return inp.strip(cutset) +@auto_convert_types def trim_left(inp: str, cutset: str) -> str: """ Returns a slice of the input with all leading Unicode code points contained in cutset removed @@ -548,6 +617,7 @@ def trim_left(inp: str, cutset: str) -> str: return inp.lstrip(cutset) +@auto_convert_types def trim_prefix(inp: str, prefix: str) -> str: """ Returns the input without the provided leading prefix string @@ -561,6 +631,7 @@ def trim_prefix(inp: str, prefix: str) -> str: return inp +@auto_convert_types def trim_right(inp: str, cutset: str) -> str: """ Returns a string, with all trailing Unicode code points contained in cutset removed @@ -572,6 +643,7 @@ def trim_right(inp: str, cutset: str) -> str: return inp.rstrip(cutset) +@auto_convert_types def trim_space(inp: str) -> str: """ Returns a string, with all leading and trailing white space removed, as defined by Unicode @@ -583,6 +655,7 @@ def trim_space(inp: str) -> str: return inp.strip() +@auto_convert_types def trim_suffix(inp: str, suffix: str) -> str: """ Returns input without the provided trailing suffix string @@ -607,6 +680,7 @@ def unix_time(optional_seconds: int = 0) -> int: return int(time.time()) + optional_seconds +@auto_convert_types def url_decode(inp: str) -> str: """ URL decodes the input string @@ -617,6 +691,7 @@ def url_decode(inp: str) -> str: return urllib.parse.unquote_plus(inp) +@auto_convert_types def url_encode(inp: str) -> str: """ URL encodes the input string @@ -640,6 +715,7 @@ def wait_for(seconds: int) -> bool: return True +@auto_convert_types def join(separator: str, *elements: str) -> str: """ Joins the given elements using the specified separator @@ -679,6 +755,7 @@ def date_time(date_time_format: str, optional_unix_time: int = int(time.time())) return datetime.datetime.utcfromtimestamp(optional_unix_time).strftime(date_time_format) +@auto_convert_types def to_unix_time(inp: str, layout: str = "%Y-%m-%d %H:%M:%S") -> int: """ Parses a string date time using default or user given layouts, then returns its Unix timestamp @@ -690,6 +767,7 @@ def to_unix_time(inp: str, layout: str = "%Y-%m-%d %H:%M:%S") -> int: return int(time.mktime(datetime.datetime.strptime(inp, layout).timetuple())) +@auto_convert_types def starts_with(inp: str, *prefix: str) -> bool: """ Checks if the string starts with any of the provided substrings @@ -701,6 +779,7 @@ def starts_with(inp: str, *prefix: str) -> bool: return any(inp.startswith(p) for p in prefix) +@auto_convert_types def line_starts_with(inp: str, *prefix: str) -> bool: """ Checks if any line of the string starts with any of the provided substrings @@ -716,6 +795,7 @@ def line_starts_with(inp: str, *prefix: str) -> bool: return False +@auto_convert_types def ends_with(inp: str, *suffix: str) -> bool: """ Checks if the string ends with any of the provided substrings @@ -727,6 +807,7 @@ def ends_with(inp: str, *suffix: str) -> bool: return any(inp.endswith(s) for s in suffix) +@auto_convert_types def line_ends_with(inp: str, *suffix: str) -> bool: """ Checks if any line of the string ends with any of the provided substrings diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/safe_eval.py b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/safe_eval.py index 1f4458af..3a0c41f0 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/safe_eval.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/safe_eval.py @@ -163,7 +163,9 @@ def safe_eval(expression, variables): logger.debug(f'[+] Expressions convert: {expression} -> {new_expression}') expression = new_expression if not _check_expression(expression, allowed_variables=list(variables.keys())): - raise Exception(f"Invalid expression [{expression}], only a very simple subset of Python is allowed.") + raise Exception( + f"Invalid expression {expression}, possibly due to unsupported functions in the template or " + "unresolved variables. If you suspect this is a Pocsuite3 issue, please submit an issue on GitHub.") return eval(expression, globals(), variables) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py index 1cd6f6a9..5e476106 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py @@ -143,25 +143,13 @@ def http_response_to_dsl_map(resp: requests.Response): return data -def http_get_match_part(part: str, resp_data: dict, interactsh=None, return_bytes: bool = False) -> str: +def http_get_match_part(part: str, resp_data: dict, return_bytes: bool = False) -> str: result = '' if part == '': part = 'body' if part in resp_data: result = resp_data[part] - elif part.startswith('interactsh'): - if not isinstance(interactsh, InteractshClient): - result = '' - # poll oob data - else: - interactsh.poll() - if part == 'interactsh_protocol': - result = '\n'.join(interactsh.interactsh_protocol) - elif part == 'interactsh_request': - result = '\n'.join(interactsh.interactsh_request) - elif part == 'interactsh_response': - result = '\n'.join(interactsh.interactsh_response) if return_bytes and not isinstance(result, bytes): result = str(result).encode() @@ -177,9 +165,19 @@ def http_match(request: HttpRequest, resp_data: dict, interactsh=None): matchers = request.matchers matchers_result = [] + if 'interactsh_' in str(matchers) and isinstance(interactsh, InteractshClient): + interactsh.poll() + resp_data['interactsh_protocol'] = '\n'.join(interactsh.interactsh_protocol) + resp_data['interactsh_request'] = '\n'.join(interactsh.interactsh_request) + resp_data['interactsh_response'] = '\n'.join(interactsh.interactsh_response) + else: + resp_data['interactsh_protocol'] = '' + resp_data['interactsh_request'] = '' + resp_data['interactsh_response'] = '' + for i, matcher in enumerate(matchers): matcher_res = False - item = http_get_match_part(matcher.part, resp_data, interactsh, matcher.type == MatcherType.BinaryMatcher) + item = http_get_match_part(matcher.part, resp_data, matcher.type == MatcherType.BinaryMatcher) if matcher.type == MatcherType.StatusMatcher: matcher_res = match_status_code(matcher, resp_data.get('status_code', 0)) @@ -288,7 +286,7 @@ def http_request_generator(request: HttpRequest, dynamic_values: OrderedDict): headers = raws[1:index - 1] headers = extract_dict('\n'.join(headers), '\n', ": ") - data = raws[index] + data = '\n'.join(raws[index:]) else: headers = extract_dict('\n'.join(raws[1:]), '\n', ": ") diff --git a/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py index 36cd5165..953bf95d 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py @@ -88,25 +88,13 @@ class NetworkRequest: read_all: bool = False -def network_get_match_part(part: str, resp_data: dict, interactsh=None, return_bytes: bool = False) -> str: +def network_get_match_part(part: str, resp_data: dict, return_bytes: bool = False) -> str: result = '' if part in ['', 'all', 'body']: part = 'data' if part in resp_data: result = resp_data[part] - elif part.startswith('interactsh'): - if not isinstance(interactsh, InteractshClient): - result = '' - # poll oob data - else: - interactsh.poll() - if part == 'interactsh_protocol': - result = '\n'.join(interactsh.interactsh_protocol) - elif part == 'interactsh_request': - result = '\n'.join(interactsh.interactsh_request) - elif part == 'interactsh_response': - result = '\n'.join(interactsh.interactsh_response) if return_bytes and not isinstance(result, bytes): result = str(result).encode() @@ -148,9 +136,19 @@ def network_match(request: NetworkRequest, resp_data: dict, interactsh=None): matchers = request.matchers matchers_result = [] + if 'interactsh_' in str(matchers) and isinstance(interactsh, InteractshClient): + interactsh.poll() + resp_data['interactsh_protocol'] = '\n'.join(interactsh.interactsh_protocol) + resp_data['interactsh_request'] = '\n'.join(interactsh.interactsh_request) + resp_data['interactsh_response'] = '\n'.join(interactsh.interactsh_response) + else: + resp_data['interactsh_protocol'] = '' + resp_data['interactsh_request'] = '' + resp_data['interactsh_response'] = '' + for i, matcher in enumerate(matchers): matcher_res = False - item = network_get_match_part(matcher.part, resp_data, interactsh, matcher.type == MatcherType.BinaryMatcher) + item = network_get_match_part(matcher.part, resp_data, matcher.type == MatcherType.BinaryMatcher) if matcher.type == MatcherType.SizeMatcher: matcher_res = match_size(matcher, len(item)) diff --git a/pocsuite3/modules/interactsh/__init__.py b/pocsuite3/modules/interactsh/__init__.py index ea28e758..a52c5b60 100644 --- a/pocsuite3/modules/interactsh/__init__.py +++ b/pocsuite3/modules/interactsh/__init__.py @@ -26,7 +26,7 @@ def __init__(self, server='', token=''): self.server = server.lstrip('.') if 'oob_server' in conf: self.server = self.server or conf.oob_server - self.server = self.server or 'interact.sh' + self.server = self.server or 'oast.me' self.token = token if 'oob_token' in conf: @@ -78,8 +78,7 @@ def poll(self): decrypt_data = self.decrypt_data(aes_key, i) result.append(decrypt_data) return result - except Exception as e: - logger.debug("[PLUGIN] Interactsh: {}".format(e)) + except Exception: count -= 1 time.sleep(1) continue From 35428091ebcc2c866554d1041164ec96b8e57c3a Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 26 Jul 2023 11:52:46 -0700 Subject: [PATCH 3/8] refactor: disable mandatory updates --- pocsuite3/lib/core/update.py | 5 ++--- pocsuite3/lib/utils/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pocsuite3/lib/core/update.py b/pocsuite3/lib/core/update.py index ff201314..0ddb9e6b 100644 --- a/pocsuite3/lib/core/update.py +++ b/pocsuite3/lib/core/update.py @@ -1,4 +1,3 @@ -import sys from pocsuite3.lib.core.data import logger, conf from six.moves.xmlrpc_client import ServerProxy from pkg_resources import parse_version @@ -47,5 +46,5 @@ def update(): ' $ unzip master.zip\n' ' $ cd pocsuite3-master\n' ' $ pip3 install -r requirements.txt\n' - ' $ python3 setup.py install\n') - sys.exit(-1) + ' $ python3 setup.py install\n' + ) diff --git a/pocsuite3/lib/utils/__init__.py b/pocsuite3/lib/utils/__init__.py index af36aac5..dd48b7f5 100644 --- a/pocsuite3/lib/utils/__init__.py +++ b/pocsuite3/lib/utils/__init__.py @@ -286,8 +286,8 @@ def minimum_version_required(ver): from pkg_resources import parse_version v1, v2 = parse_version(ver), parse_version(__version__) if v1 > v2: - logger.error(f'The minimum version required for this PoC plugin is {ver}, ' - f'you installed {__version__}, please upgrade pocsuite3 :)') + logger.warning(f'The minimum version required for this PoC plugin is {ver}, ' + f'you installed {__version__}, please consider upgrading pocsuite3.') from pocsuite3.lib.core.data import conf from pocsuite3.lib.core.update import update conf.update_all = True From 08b174926c7bb2540d79f53e9c21c1582d53d585 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 26 Jul 2023 12:59:43 -0700 Subject: [PATCH 4/8] refactor: making mmh3 an optional dependency --- .../protocols/common/expressions/__init__.py | 22 ++++++++++++++----- requirements.txt | 1 - setup.py | 4 ++-- tests/test_nuclei_helper_functions.py | 4 ++-- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py index 2c535172..c9afd97f 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py @@ -16,7 +16,6 @@ from typing import get_type_hints, Union import chardet -import mmh3 as py_mmh3 from pkg_resources import parse_version from pocsuite3.lib.core.log import LOGGER as logger from pocsuite3.lib.yaml.nuclei.protocols.common.expressions.safe_eval import safe_eval @@ -120,9 +119,11 @@ def base64_py(src: Union[bytes, str]) -> str: Example: Input: base64_py("Hello") - Output: SGVsbG8= + Output: "SGVsbG8=\n" """ - return base64(src) + if not isinstance(src, bytes): + src = src.encode('utf-8') + return py_built_in_base64.encodebytes(src).decode('utf-8') @auto_convert_types @@ -373,15 +374,24 @@ def md5(inp: Union[str, bytes]) -> str: @auto_convert_types -def mmh3(inp: str) -> int: +def mmh3(inp: str) -> str: """ Calculates the MMH3 (MurmurHash3) hash of an input Example: Input: mmh3("Hello") - Output: 316307400 + Output: "316307400" """ - return py_mmh3.hash(inp) + + try: + import mmh3 as py_mmh3 + except ImportError: + logger.error('Python extension for MurmurHash (MurmurHash3) is not installed. ' + 'Reason: https://github.com/knownsec/pocsuite3/issues/359, ' + 'You can locate the packages here: https://pypi.org/project/mmh3/') + return "0" + + return str(py_mmh3.hash(inp)) def print_debug(*args) -> None: diff --git a/requirements.txt b/requirements.txt index 679d11c0..d8562b3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,3 @@ pycryptodomex >= 3.9.0 dacite >= 1.6.0 PyYAML >= 6.0 lxml >= 4.6.0 -mmh3 >= 3.0.0 diff --git a/setup.py b/setup.py index d8afe8a4..7764fafa 100644 --- a/setup.py +++ b/setup.py @@ -57,12 +57,12 @@ def find_packages(where='.'): "dacite", "PyYAML", "lxml", - "mmh3" ], extras_require={ 'complete': [ 'pyOpenSSL', - 'jq' + 'jq', + 'mmh3' ], } ) diff --git a/tests/test_nuclei_helper_functions.py b/tests/test_nuclei_helper_functions.py index 00cd54ae..9bfea58a 100644 --- a/tests/test_nuclei_helper_functions.py +++ b/tests/test_nuclei_helper_functions.py @@ -10,7 +10,7 @@ def test_base64_decode(self): self.assertEqual(base64_decode("SGVsbG8="), b"Hello") def test_base64_py(self): - self.assertEqual(base64_py("Hello"), "SGVsbG8=") + self.assertEqual(base64_py("Hello"), "SGVsbG8=\n") def test_concat(self): self.assertEqual(concat("Hello", 123, "world"), "Hello123world") @@ -74,7 +74,7 @@ def test_md5(self): self.assertEqual(md5("Hello"), "8b1a9953c4611296a827abf8c47804d7") def test_mmh3(self): - self.assertEqual(mmh3("Hello"), 316307400) + self.assertEqual(mmh3("Hello"), "316307400") def test_rand_base(self): self.assertRegex(rand_base(5, "abc"), r"[abc]{5}") From c34ecb4bb6103d4ab9f8e3900670c933de417aab Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 26 Jul 2023 13:11:15 -0700 Subject: [PATCH 5/8] chore: bump version to 2.0.5 --- .github/workflows/release.yml | 2 +- CHANGELOG.md | 7 +++++++ manpages/poc-console.1 | 6 +++--- manpages/pocsuite.1 | 6 +++--- pocsuite3/__init__.py | 2 +- setup.py | 2 +- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 54498058..d2af82b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,7 @@ jobs: run: | export VERSION=$(echo $GH_REF | sed 's:refs/tags/v::') git config --global user.email "abcnsxyz@gmail.com" - git config --global user.name 'Tian Qiao' + git config --global user.name '13ph03nix' git commit -a -m "Version ${VERSION} (automated version bump)" git push origin master env: diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d1da182..85ded41a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# version 2.0.5 +---------------- +* fix hook failure due to urllib3 update #368 #373 +* optimize DSL expression execution #372 +* making mmh3 an optional dependency #359 +* disable mandatory updates + # version 2.0.4 ---------------- * Updated protocol names that are compatible with Nuclei v2.9.1 diff --git a/manpages/poc-console.1 b/manpages/poc-console.1 index 550ef9e1..47e0331f 100644 --- a/manpages/poc-console.1 +++ b/manpages/poc-console.1 @@ -2,7 +2,7 @@ .\" .\" Nov 3, 2022 .\" Man page author: -.\" Tian Qiao +.\" 13ph03nix .\" .SH NAME .I poc-console @@ -31,7 +31,7 @@ is maintained at: .I https://pocsuite.org .PP .SH VERSION -This manual page documents pocsuite3 version 2.0.4 +This manual page documents pocsuite3 version 2.0.5 .SH AUTHOR .br (c) 2014-present by Knownsec 404 Team @@ -46,7 +46,7 @@ redistribute this software under certain conditions. If you wish to embed pocsuite3 technology into proprietary software, we sell alternative licenses (contact 404-team@knownsec.com). .PP -Manual page started by Tian Qiao +Manual page started by 13ph03nix .PP diff --git a/manpages/pocsuite.1 b/manpages/pocsuite.1 index 46347c29..c6289b44 100644 --- a/manpages/pocsuite.1 +++ b/manpages/pocsuite.1 @@ -2,7 +2,7 @@ .\" .\" Nov 3, 2022 .\" Man page author: -.\" Tian Qiao +.\" 13ph03nix .\" .SH NAME .I pocsuite3 @@ -289,7 +289,7 @@ is maintained at: .I https://pocsuite.org .PP .SH VERSION -This manual page documents pocsuite3 version 2.0.4 +This manual page documents pocsuite3 version 2.0.5 .SH AUTHOR .br (c) 2014-present by Knownsec 404 Team @@ -304,7 +304,7 @@ redistribute this software under certain conditions. If you wish to embed pocsuite3 technology into proprietary software, we sell alternative licenses (contact 404-team@knownsec.com). .PP -Manual page started by Tian Qiao +Manual page started by 13ph03nix .PP diff --git a/pocsuite3/__init__.py b/pocsuite3/__init__.py index 37de1cea..f38c404d 100644 --- a/pocsuite3/__init__.py +++ b/pocsuite3/__init__.py @@ -1,5 +1,5 @@ __title__ = 'pocsuite3' -__version__ = '2.0.4' +__version__ = '2.0.5' __author__ = 'Knownsec 404 Team' __author_email__ = '404-team@knownsec.com' __license__ = 'GPLv2' diff --git a/setup.py b/setup.py index 7764fafa..72058dc2 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def find_packages(where='.'): setup( name='pocsuite3', - version='2.0.4', + version='2.0.5', url='https://pocsuite.org', description='Open-sourced remote vulnerability testing framework.', long_description=long_description, From f3086cb8449178b5387f3bf74d55bcb413b912d2 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:04:10 -0700 Subject: [PATCH 6/8] fix: flake8 error --- pocsuite3/lib/utils/__init__.py | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pocsuite3/lib/utils/__init__.py b/pocsuite3/lib/utils/__init__.py index dd48b7f5..375ac169 100644 --- a/pocsuite3/lib/utils/__init__.py +++ b/pocsuite3/lib/utils/__init__.py @@ -287,7 +287,7 @@ def minimum_version_required(ver): v1, v2 = parse_version(ver), parse_version(__version__) if v1 > v2: logger.warning(f'The minimum version required for this PoC plugin is {ver}, ' - f'you installed {__version__}, please consider upgrading pocsuite3.') + f'you installed {__version__}, please consider upgrading pocsuite3.') from pocsuite3.lib.core.data import conf from pocsuite3.lib.core.update import update conf.update_all = True diff --git a/requirements.txt b/requirements.txt index d8562b3d..679d11c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ pycryptodomex >= 3.9.0 dacite >= 1.6.0 PyYAML >= 6.0 lxml >= 4.6.0 +mmh3 >= 3.0.0 From 8ac5e5cb821ac78bfc1e4498d179dbe17cd8dacd Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:10:56 -0700 Subject: [PATCH 7/8] chore: update testcase --- tests/test_api_diy_options.py | 2 ++ tests/test_request_raw.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tests/test_api_diy_options.py b/tests/test_api_diy_options.py index f39c5938..e9b5cc75 100644 --- a/tests/test_api_diy_options.py +++ b/tests/test_api_diy_options.py @@ -27,6 +27,7 @@ def verify_result(self): result = get_results().pop() self.assertTrue(result.status == 'success') + @unittest.skip(reason='significant latency') def test_cookie(self): config = { 'url': ['http://httpbin.org/post'], @@ -42,6 +43,7 @@ def test_cookie(self): result = get_results().pop() self.assertTrue(result.status == 'success') + @unittest.skip(reason='significant latency') def test_cookie_dict_params(self): config = { 'url': ['http://httpbin.org/post'], diff --git a/tests/test_request_raw.py b/tests/test_request_raw.py index 7520ff54..562f5f77 100644 --- a/tests/test_request_raw.py +++ b/tests/test_request_raw.py @@ -9,6 +9,7 @@ def setUp(self): def tearDown(self): pass + @unittest.skip(reason='significant latency') def test_get(self): raw = ''' GET /get?a=1&b=2 HTTP/1.1 @@ -24,6 +25,7 @@ def test_get(self): r = requests.httpraw(raw) self.assertTrue(r.json()['args'] == {'a': '1', 'b': '2'}) + @unittest.skip(reason='significant latency') def test_post(self): raw = ''' POST /post HTTP/1.1 @@ -41,6 +43,7 @@ def test_post(self): r = requests.httpraw(raw) self.assertTrue(r.json()['data'] == 'a=1&b=2') + @unittest.skip(reason='significant latency') def test_json(self): raw = ''' POST /post HTTP/1.1 From 0338e567c7650ecd56f79f60d761455e309580a1 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:12:45 -0700 Subject: [PATCH 8/8] fix: flake8 error --- tests/test_request_raw.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_request_raw.py b/tests/test_request_raw.py index 562f5f77..c276d364 100644 --- a/tests/test_request_raw.py +++ b/tests/test_request_raw.py @@ -37,7 +37,7 @@ def test_post(self): Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9,en;q=0.8 Cookie: _gauges_unique_hour=1; _gauges_unique_day=1; _gauges_unique_month=1; _gauges_unique_year=1; _gauges_unique=1 - + a=1&b=2 ''' r = requests.httpraw(raw) @@ -55,7 +55,7 @@ def test_json(self): Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9,en;q=0.8 Cookie: _gauges_unique_hour=1; _gauges_unique_day=1; _gauges_unique_month=1; _gauges_unique_year=1; _gauges_unique=1 - + {"pocsuite":"v3.0"} ''' r = requests.httpraw(raw)