From 039219ab9ff8714bd905bd6bdcb5191352d0d2af Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 16:52:35 -0400 Subject: [PATCH 01/12] state_set: Improve `state_set_search` performance, correct result. The description says "Return where an item would be, if it were inserted", but it was returning the last element <= rather than the first element >=, then the call to `state_set_cmpval` later was shifting i by 1 for that specific case. Handle it correctly inside the search function instead. Two other all call sites need to check whether the result refers to the append position (one past the end of the array) before checking `set->a[i] == state`, update them. Add a fast path upfront: It's VERY common to append states in order to the state array, so before we binary search each first compare against the last entry (unless empty). --- src/adt/stateset.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index c1cff9933..4d4de9438 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -138,7 +138,8 @@ state_set_cmp(const struct state_set *a, const struct state_set *b) } /* - * Return where an item would be, if it were inserted + * Return where an item would be, if it were inserted. + * When insertion would append this returns one past the array. */ static size_t state_set_search(const struct state_set *set, fsm_state_t state) @@ -150,6 +151,11 @@ state_set_search(const struct state_set *set, fsm_state_t state) assert(!IS_SINGLETON(set)); assert(set->a != NULL); + /* fast path: append case */ + if (set->i > 0 && state > set->a[set->i - 1]) { + return set->i; + } + start = mid = 0; end = set->i; @@ -161,6 +167,12 @@ state_set_search(const struct state_set *set, fsm_state_t state) end = mid; } else if (r > 0) { start = mid + 1; + /* update mid if we're about to halt, because + * we're looking for the first position >= state, + * not the last position <= */ + if (start == end) { + mid = start; + } } else { return mid; } @@ -242,7 +254,7 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, */ if (!state_set_empty(set)) { i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { return 1; } } @@ -261,9 +273,6 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->n *= 2; } - if (state_set_cmpval(state, set->a[i]) > 0) { - i++; - } if (i <= set->i) { memmove(&set->a[i + 1], &set->a[i], (set->i - i) * (sizeof *set->a)); @@ -470,7 +479,7 @@ state_set_remove(struct state_set **setp, fsm_state_t state) } i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { if (i < set->i) { memmove(&set->a[i], &set->a[i + 1], (set->i - i - 1) * (sizeof *set->a)); } @@ -524,7 +533,7 @@ state_set_contains(const struct state_set *set, fsm_state_t state) } i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { return 1; } From 4bb505dfc1da3d5b42ee7b886c57aaeb58cb1d37 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 16:57:13 -0400 Subject: [PATCH 02/12] stateset: Avoid memmove of size 0. --- src/adt/stateset.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 4d4de9438..645424555 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -273,8 +273,7 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->n *= 2; } - - if (i <= set->i) { + if (i < set->i) { memmove(&set->a[i + 1], &set->a[i], (set->i - i) * (sizeof *set->a)); } From 0e4d46aa5e73da43cc53c864c33ced1d42a720cd Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:09:01 -0400 Subject: [PATCH 03/12] stateset: Add note about potentially expensive assertion. In -O0 this can become pretty expensive (~25% of overall runtime for `time ./re -rpcre -C '^[ab]{0,2000}$'`), but when built with -O3 very little overhead remains. I'm adding this comment because every time I see this it seems to me like it should have `EXPENSIVE_CHECKS` around it, but profiling is telling me it really doesn't matter. --- src/adt/stateset.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 645424555..7839234d2 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -284,6 +284,8 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->i = 1; } + /* This assert can be pretty expensive in -O0 but in -O3 it has very + * little impact on the overall runtime. */ assert(state_set_contains(set, state)); return 1; From 351cea042ed0993868739260532af9f924a92ce2 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:11:07 -0400 Subject: [PATCH 04/12] stateset: Comment struct fields. --- src/adt/stateset.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 7839234d2..8d73a9b5f 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -44,8 +44,8 @@ struct state_set { const struct fsm_alloc *alloc; fsm_state_t *a; - size_t i; - size_t n; + size_t i; /* used */ + size_t n; /* ceil */ }; int From be9bd88cc7f6d1b3b562d12c118fad82daba21e8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:12:04 -0400 Subject: [PATCH 05/12] edgeset: Fix indentation for `#if`'d block. --- src/adt/edgeset.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index c718727ca..f1a9e9f31 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -223,9 +223,9 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, assert(set->count <= set->ceil); #if LOG_BITSET - fprintf(stderr, " -- edge_set_add: symbols [0x%lx, 0x%lx, 0x%lx, 0x%lx] -> state %d on %p\n", - symbols[0], symbols[1], symbols[2], symbols[3], - state, (void *)set); + fprintf(stderr, " -- edge_set_add: symbols [0x%lx, 0x%lx, 0x%lx, 0x%lx] -> state %d on %p\n", + symbols[0], symbols[1], symbols[2], symbols[3], + state, (void *)set); #endif /* Linear search for a group with the same destination From 7a47e2a66946e7e53aafaeb6cb7e45b0904b86f6 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:12:31 -0400 Subject: [PATCH 06/12] edgeset: Switch from linear to binary searching in edge_set_add_bulk. This is a major hotspot when doing epsilon removal over large runs of potentially skipped states (as might appear from `^[ab]{0,2000}$`). Add a fast path for appending, which is also very common. Extract the edge set destination search into its own function, `find_state_position`, and add a `#define` to switch between linear search, binary search, or calling both and comparing the result. I will remove linear search in the next commit, but am checking this in as an intermediate step for checking & benchmarking. --- src/adt/edgeset.c | 182 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 165 insertions(+), 17 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index f1a9e9f31..818b78493 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -11,6 +11,7 @@ #include #define LOG_BITSET 0 +#define LOG_BSEARCH 0 #include "libfsm/internal.h" /* XXX: for allocating struct fsm_edge, and the edges array */ @@ -184,6 +185,157 @@ edge_set_advise_growth(struct edge_set **pset, const struct fsm_alloc *alloc, return 1; } +enum fsp_res { + FSP_FOUND_INSERT_POSITION, + FSP_FOUND_VALUE_PRESENT, +}; + +/* Use binary search to find the first position N where set->groups[N].to >= state, + * which includes the position immediately following the last entry. Return an enum + * which indicates whether state is already present. */ +static enum fsp_res +find_state_position_bsearch(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + size_t lo = 0, hi = set->count; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: looking for %d in %p (count %zu)\n", + __func__, state, (void *)set, set->count); + } + +#if EXPENSIVE_CHECKS + /* invariant: input is unique and sorted */ + for (size_t i = 1; i < set->count; i++) { + assert(set->groups[i - 1].to < set->groups[i].to); + } +#endif + + if (set->count == 0) { + if (LOG_BSEARCH) { + fprintf(stderr, "%s: empty, returning 0\n", __func__); + } + *dst = 0; + return FSP_FOUND_INSERT_POSITION; + } else { + if (LOG_BSEARCH) { + fprintf(stderr, "%s: fast path: looking for %d, set->groups[last].to %d\n", + __func__, state, set->groups[hi - 1].to); + } + + /* Check the last entry so we can append in constant time. */ + const fsm_state_t last = set->groups[hi - 1].to; + if (state > last) { + *dst = hi; + return FSP_FOUND_INSERT_POSITION; + } else if (state == last) { + *dst = hi - 1; + return FSP_FOUND_VALUE_PRESENT; + } + } + + size_t mid; + while (lo < hi) { /* lo <= mid < hi */ + mid = lo + (hi - lo)/2; /* avoid overflow */ + const struct edge_group *eg = &set->groups[mid]; + const fsm_state_t cur = eg->to; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: lo %zu, hi %zu, mid %zu, cur %d, looking for %d\n", + __func__, lo, hi, mid, cur, state); + } + + if (state == cur) { + *dst = mid; + return FSP_FOUND_VALUE_PRESENT; + } else if (state > cur) { + lo = mid + 1; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: new lo %zd\n", __func__, lo); + } + + /* Update mid if we're about to halt, because we're looking + * for the first position >= state, not the last position <=. */ + if (lo == hi) { + mid = lo; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: special case, updating mid to %zd\n", __func__, mid); + } + } + } else if (state < cur) { + hi = mid; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: new hi %zd\n", __func__, hi); + } + } + } + + if (LOG_BSEARCH) { + fprintf(stderr, "%s: halting at %zd (looking for %d, cur %d)\n", + __func__, mid, state, set->groups[mid].to); + } + + /* dst is now the first position > state (== case is handled above), + * which may be one past the end of the array. */ + assert(mid == set->count || set->groups[mid].to > state); + *dst = mid; + return FSP_FOUND_INSERT_POSITION; +} + +static enum fsp_res +find_state_position_linear(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + /* Linear search for a group with the same destination + * state, or the position where that group would go. */ + size_t i; + for (i = 0; i < set->count; i++) { + const struct edge_group *eg = &set->groups[i]; + if (eg->to == state) { + *dst = i; + return FSP_FOUND_VALUE_PRESENT; + } else if (eg->to > state) { + break; /* will shift down and insert below */ + } else { + continue; + } + } + + *dst = i; + return FSP_FOUND_INSERT_POSITION; +} + +/* Find the state in the edge set, or where it would be inserted if not present. */ +static enum fsp_res +find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + /* 0: linear, 1: bsearch, -1: call both, to check result */ +#define USE_BSEARCH 1 + + switch (USE_BSEARCH) { + case 0: + return find_state_position_linear(set, state, dst); + case 1: + return find_state_position_bsearch(set, state, dst); + case -1: + { + size_t dst_linear, dst_bsearch; + enum fsp_res res_linear = find_state_position_linear(set, state, &dst_linear); + enum fsp_res res_bsearch = find_state_position_bsearch(set, state, &dst_bsearch); + + if (res_linear != res_bsearch || dst_linear != dst_bsearch) { + fprintf(stderr, "%s: disagreement for state %d: linear res %d, dst %zu, bsearch res %d, dst %zu\n", + __func__, state, + res_linear, dst_linear, + res_bsearch, dst_bsearch); + for (size_t i = 0; i < set->count; i++) { + fprintf(stderr, "set->groups[%zu].to: %d\n", i, set->groups[i].to); + } + } + assert(res_linear == res_bsearch); + assert(dst_linear == dst_bsearch); + *dst = dst_linear; + return res_linear; + } + } +} + int edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, uint64_t symbols[256/64], fsm_state_t state) @@ -228,25 +380,21 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, state, (void *)set); #endif - /* Linear search for a group with the same destination - * state, or the position where that group would go. */ - for (i = 0; i < set->count; i++) { + switch (find_state_position(set, state, &i)) { + case FSP_FOUND_VALUE_PRESENT: + assert(i < set->count); + /* This API does not indicate whether that + * symbol -> to edge was already present. */ eg = &set->groups[i]; - - if (eg->to == state) { - /* This API does not indicate whether that - * symbol -> to edge was already present. */ - size_t i; - for (i = 0; i < 256/64; i++) { - eg->symbols[i] |= symbols[i]; - } - dump_edge_set(set); - return 1; - } else if (eg->to > state) { - break; /* will shift down and insert below */ - } else { - continue; + for (i = 0; i < 256/64; i++) { + eg->symbols[i] |= symbols[i]; } + dump_edge_set(set); + return 1; + + break; + case FSP_FOUND_INSERT_POSITION: + break; /* continue below */ } /* insert/append at i */ From bca248825b49f56cbea4421b8bdd9b118e5468e7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:19:24 -0400 Subject: [PATCH 07/12] edgeset: Commit to using binary search. When I run `time ./re -rpcre -C '^[ab]{0,2000}$'` locally for -O3: - linear search: 2.991s - binary search: 1.521s --- src/adt/edgeset.c | 59 +---------------------------------------------- 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index 818b78493..dd3b8853c 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -194,7 +194,7 @@ enum fsp_res { * which includes the position immediately following the last entry. Return an enum * which indicates whether state is already present. */ static enum fsp_res -find_state_position_bsearch(const struct edge_set *set, fsm_state_t state, size_t *dst) +find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) { size_t lo = 0, hi = set->count; if (LOG_BSEARCH) { @@ -279,63 +279,6 @@ find_state_position_bsearch(const struct edge_set *set, fsm_state_t state, size_ return FSP_FOUND_INSERT_POSITION; } -static enum fsp_res -find_state_position_linear(const struct edge_set *set, fsm_state_t state, size_t *dst) -{ - /* Linear search for a group with the same destination - * state, or the position where that group would go. */ - size_t i; - for (i = 0; i < set->count; i++) { - const struct edge_group *eg = &set->groups[i]; - if (eg->to == state) { - *dst = i; - return FSP_FOUND_VALUE_PRESENT; - } else if (eg->to > state) { - break; /* will shift down and insert below */ - } else { - continue; - } - } - - *dst = i; - return FSP_FOUND_INSERT_POSITION; -} - -/* Find the state in the edge set, or where it would be inserted if not present. */ -static enum fsp_res -find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) -{ - /* 0: linear, 1: bsearch, -1: call both, to check result */ -#define USE_BSEARCH 1 - - switch (USE_BSEARCH) { - case 0: - return find_state_position_linear(set, state, dst); - case 1: - return find_state_position_bsearch(set, state, dst); - case -1: - { - size_t dst_linear, dst_bsearch; - enum fsp_res res_linear = find_state_position_linear(set, state, &dst_linear); - enum fsp_res res_bsearch = find_state_position_bsearch(set, state, &dst_bsearch); - - if (res_linear != res_bsearch || dst_linear != dst_bsearch) { - fprintf(stderr, "%s: disagreement for state %d: linear res %d, dst %zu, bsearch res %d, dst %zu\n", - __func__, state, - res_linear, dst_linear, - res_bsearch, dst_bsearch); - for (size_t i = 0; i < set->count; i++) { - fprintf(stderr, "set->groups[%zu].to: %d\n", i, set->groups[i].to); - } - } - assert(res_linear == res_bsearch); - assert(dst_linear == dst_bsearch); - *dst = dst_linear; - return res_linear; - } - } -} - int edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, uint64_t symbols[256/64], fsm_state_t state) From bd5398d624482b3715ba384e1a95084d1c18bc90 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:08:44 -0400 Subject: [PATCH 08/12] determinise: Drastically reduce calls to qsort. After the other changes in this PR, calls to qsort from `sort_and_dedup_dst_buf` are one of the largest remaining hotspots in the profile. We can often avoid calling qsort, though: - If there is <= 1 entry, just return, it's sorted. - Otherwise, first do a sweep through the array noting the min and max values. Unless there is a huge range between them, it's much faster to build a bitset from them in a small (max 10KB) stack-allocated array and then unpack the bitset (now sorted and unique). Only the needed portion of the array is initialized. I have not done a lot of experimentation to find a cutoff point where the bitset becomes slower than qsort (it may be much larger), I picked 10KB because it's likely to be safe to stack-allocate. I tried changing the bitset unpacking to use an 8 or 16 bit mask and jump forward faster through large sub-word ranges of 0 bits, but any improvement was lost among random variation, so I decided it wasn't worth the extra complexity. We already skip whole words that are 0. --- src/libfsm/determinise.c | 95 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 18 deletions(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 56e135afd..ac15f05a8 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -2016,28 +2016,87 @@ static void sort_and_dedup_dst_buf(fsm_state_t *buf, size_t *used) { const size_t orig_used = *used; - qsort(buf, orig_used, sizeof(buf[0]), cmp_fsm_state_t); - - /* squash out duplicates */ - size_t rd = 1; - size_t wr = 1; - while (rd < orig_used) { - if (buf[rd - 1] == buf[rd]) { - rd++; /* skip */ - } else { - buf[wr] = buf[rd]; - rd++; - wr++; - } + + if (orig_used <= 1) { + return; /* no change */ } - *used = wr; -#if EXPENSIVE_CHECKS - assert(wr <= orig_used); - for (size_t i = 1; i < *used; i++) { - assert(buf[i - 1] < buf[i]); + /* Figure out what the min and max values are, because + * when the difference between them is not too large it + * can be significantly faster to avoid qsort here. */ + fsm_state_t min = (fsm_state_t)-1; + fsm_state_t max = 0; + for (size_t i = 0; i < orig_used; i++) { + const fsm_state_t cur = buf[i]; + if (cur < min) { min = cur; } + if (cur > max) { max = cur; } } + + /* If there's only one unique value, then we're done. */ + if (min == max) { + buf[0] = min; + *used = 1; + return; + } + +/* 81920 = 10 KB buffer on the stack. This must be divisible by 64. + * Set to 0 to disable. */ +#define QSORT_CUTOFF 81920 + + if (QSORT_CUTOFF == 0 || max - min > QSORT_CUTOFF) { + /* If the bitset would be very large but sparse due to + * extreme values, then fall back on using qsort and + * then sweeping over the array to squash out + * duplicates. */ + qsort(buf, orig_used, sizeof(buf[0]), cmp_fsm_state_t); + + /* squash out duplicates */ + size_t rd = 1; + size_t wr = 1; + while (rd < orig_used) { + if (buf[rd - 1] == buf[rd]) { + rd++; /* skip */ + } else { + buf[wr] = buf[rd]; + rd++; + wr++; + } + } + + *used = wr; +#if EXPENSIVE_CHECKS + assert(wr <= orig_used); + for (size_t i = 1; i < *used; i++) { + assert(buf[i - 1] < buf[i]); + } #endif + } else { + /* Convert the array into a bitset and back, which sorts + * and deduplicates in the process. Add 1 to avoid a zero- + * zero-length array error if QSORT_CUTOFF is 0. */ + uint64_t bitset[QSORT_CUTOFF/64 + 1]; + const size_t words = u64bitset_words(max - min); + memset(bitset, 0x00, words * sizeof(bitset[0])); + + for (size_t i = 0; i < orig_used; i++) { + u64bitset_set(bitset, buf[i] - min); + } + + size_t dst = 0; + for (size_t i = 0; i < words; i++) { + const uint64_t w = bitset[i]; + if (w != 0) { /* skip empty words */ + uint64_t bit = 0x1; + for (size_t b_i = 0; b_i < 64; b_i++, bit <<= 1) { + if (w & bit) { + buf[dst] = 64*i + b_i + min; + dst++; + } + } + } + } + *used = dst; + } } static int From 70dc1b319c370f7ebfeb3fa6372ba82a0ffec9b8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:17:56 -0400 Subject: [PATCH 09/12] edgeset: Remove stale comment. --- src/adt/edgeset.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index dd3b8853c..9658213c8 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -326,8 +326,6 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, switch (find_state_position(set, state, &i)) { case FSP_FOUND_VALUE_PRESENT: assert(i < set->count); - /* This API does not indicate whether that - * symbol -> to edge was already present. */ eg = &set->groups[i]; for (i = 0; i < 256/64; i++) { eg->symbols[i] |= symbols[i]; From ae7c5b748bf20edfd77a858b17470c092e97c3df Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:46:35 -0400 Subject: [PATCH 10/12] UBSan: Avoid implicit signed/unsigned conversion. --- src/libre/ast_compile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index 502faf8b4..aa0902d55 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -208,11 +208,11 @@ addedge_literal(struct comp_env *env, enum re_flags re_flags, assert(to < env->fsm->statecount); if (re_flags & RE_ICASE) { - if (!fsm_addedge_literal(fsm, from, to, tolower((unsigned char) c))) { + if (!fsm_addedge_literal(fsm, from, to, (char)tolower((unsigned char) c))) { return 0; } - if (!fsm_addedge_literal(fsm, from, to, toupper((unsigned char) c))) { + if (!fsm_addedge_literal(fsm, from, to, (char)toupper((unsigned char) c))) { return 0; } } else { From f623b5ef14bac3461fbeeefc98587a1bdceab350 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:49:22 -0400 Subject: [PATCH 11/12] UBSan: Avoid implicit signed/unsigned conversion. --- src/retest/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/retest/main.c b/src/retest/main.c index b6b4c52f7..e01c93e7c 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -393,7 +393,7 @@ parse_escapes(char *s, char **errpos, int *lenp) ndig++; } else { - s[j++] = ccode; + s[j++] = (char)ccode; st = ST_BARE; if (!hexcurly) { From a58cab2ba1fd5ff0d5b8eb55672559b1ed136560 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 13:17:12 -0400 Subject: [PATCH 12/12] bugfix: The range is min..max inclusive, so add 1. If min and max are exactly 64 states apart the upper value was getting silently dropped due to an incorrect `words` value here. One of the patterns in the PCRE suite triggers this: ./re -rpcre '(?:c|d)(?:)(?:aaaaaaaa(?:)(?:bbbbbbbb)(?:bbbbbbbb(?:))(?:bbbbbbbb(?:)(?:bbbbbbbb)))' "caaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" This should match, but did not. --- src/libfsm/determinise.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index ac15f05a8..3dbdc429e 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -2075,7 +2075,7 @@ sort_and_dedup_dst_buf(fsm_state_t *buf, size_t *used) * and deduplicates in the process. Add 1 to avoid a zero- * zero-length array error if QSORT_CUTOFF is 0. */ uint64_t bitset[QSORT_CUTOFF/64 + 1]; - const size_t words = u64bitset_words(max - min); + const size_t words = u64bitset_words(max - min + 1); memset(bitset, 0x00, words * sizeof(bitset[0])); for (size_t i = 0; i < orig_used; i++) {