Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Unicode and Windows support #20

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
+273 −71
Open

Add Unicode and Windows support #20

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6272927
Add unicode character support for levenshtein and osa
LEFTazs Feb 3, 2020
9d64fb8
Add Windows and unicode support for damerau
LEFTazs Feb 3, 2020
58a00ef
Remove length arguments from the C methods
LEFTazs Feb 4, 2020
530042b
Fix negative indexing error
LEFTazs Feb 5, 2020
c0d9b33
Add legacy code for backwards compatibility
LEFTazs Feb 6, 2020
454af44
Refractor legacy code
LEFTazs Feb 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions test/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,3 +183,61 @@ def test_dl(self):
self.assertEqual(dam_lev('bca', 'ab'), 2)
self.assertEqual(dam_lev('ab', 'bdca'), 3)
self.assertEqual(dam_lev('bdca', 'ab'), 3)


class TestClevWithUnicode(unittest.TestCase):

def setUp(self):
self.iw = np.ones(10001, dtype=np.float64)
self.dw = np.ones(10001, dtype=np.float64)
self.sw = np.ones((10001, 10001), dtype=np.float64)
self.tw = np.ones((10001, 10001), dtype=np.float64)
self.iw[ord("á")] = 2.0
self.dw[ord("á")] = 2.0
self.iw[ord("ő")] = 9.0
self.dw[ord("ő")] = 9.0
self.iw[ord("Ұ")] = 10.0
self.dw[ord("Ұ")] = 10.0


def _lev(self, x, y):
return lev(x, y, self.iw, self.dw, self.sw)

def _osa(self, x, y):
return osa(x, y, self.iw, self.dw, self.sw, self.tw)

def _dl(self, x, y):
return dam_lev(x, y, self.iw, self.dw, self.sw, self.tw)

def test_lev(self):
try:
self.assertEqual(self._lev('átívelődök', 'átívelődök'), 0.0)
self.assertEqual(self._lev('', 'átívelődök'), 19.0)
self.assertEqual(self._lev('átívelődök', ''), 19.0)
self.assertEqual(self._lev('', ''), 0.0)
self.assertEqual(self._lev('átívelődök', 'átívelőd'), 2.0)
self.assertEqual(self._lev('', 'ҰǴʚΏ¤☣✐'), 16.0)
except UnicodeEncodeError:
self.fail("Could not handle special characters")

def test_osa(self):
try:
self.assertEqual(self._osa('átívelődök', 'átívelődök'), 0.0)
self.assertEqual(self._osa('', 'átívelődök'), 19.0)
self.assertEqual(self._osa('átívelődök', ''), 19.0)
self.assertEqual(self._osa('', ''), 0.0)
self.assertEqual(self._osa('átívelődök', 'átívelőd'), 2.0)
self.assertEqual(self._osa('', 'ҰǴʚΏ¤☣✐'), 16.0)
except UnicodeEncodeError:
self.fail("Could not handle special characters")

def test_dl(self):
try:
self.assertEqual(self._dl('átívelődök', 'átívelődök'), 0.0)
self.assertEqual(self._dl('', 'átívelődök'), 19.0)
self.assertEqual(self._dl('átívelődök', ''), 19.0)
self.assertEqual(self._dl('', ''), 0.0)
self.assertEqual(self._dl('átívelődök', 'átívelőd'), 2.0)
self.assertEqual(self._dl('', 'ҰǴʚΏ¤☣✐'), 16.0)
except UnicodeEncodeError:
self.fail("Could not handle special characters")
20 changes: 7 additions & 13 deletions weighted_levenshtein/clev.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,36 +3,30 @@ from libc.float cimport DBL_MAX as DTYPE_MAX
ctypedef double DTYPE_t

cdef enum:
ALPHABET_SIZE = 128
ALPHABET_SIZE = 512


cdef DTYPE_t c_damerau_levenshtein(
unsigned char* str_a,
Py_ssize_t len_a,
unsigned char* str_b,
Py_ssize_t len_b,
int[:] str_a,
LEFTazs marked this conversation as resolved.
Show resolved Hide resolved
int[:] str_b,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
DTYPE_t[:,::1] transpose_costs) nogil


cdef DTYPE_t c_optimal_string_alignment(
unsigned char* word_m,
Py_ssize_t m,
unsigned char* word_n,
Py_ssize_t n,
int[:] word_m,
int[:] word_n,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
DTYPE_t[:,::1] transpose_costs) nogil


cdef DTYPE_t c_levenshtein(
unsigned char* word_m,
Py_ssize_t m,
unsigned char* word_n,
Py_ssize_t n,
int[:] word_m,
int[:] word_n,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs) nogil
Expand Down
101 changes: 61 additions & 40 deletions weighted_levenshtein/clev.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
#!python
# cython: language_level=3, boundscheck=False, wraparound=False, embedsignature=True, linetrace=True, c_string_type=str, c_string_encoding=ascii
# distutils: define_macros=CYTHON_TRACE_NOGIL=1
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why was CYTHON_TRACE_NOGIL removed?

# cython: language_level=3, boundscheck=False, wraparound=False, embedsignature=True, linetrace=True, c_string_type=unicode, c_string_encoding=utf8

from libc.stdlib cimport malloc, free
from cython.view cimport array as cvarray
from .clev cimport DTYPE_t, DTYPE_MAX, ALPHABET_SIZE
import numpy as np


cyarr = cvarray(shape=(ALPHABET_SIZE,), itemsize=sizeof(double), format="d")
Expand Down Expand Up @@ -132,12 +132,29 @@ cdef inline DTYPE_t row_insert_range_cost(

# End Array2D

def convert_string_to_int_array(char* str):
LEFTazs marked this conversation as resolved.
Show resolved Hide resolved
s = str.encode('utf-8').decode('utf-8')
intarr = np.empty([len(s), ], dtype=int)
for i, c in enumerate(s):
intarr[i] = ord(c)
return intarr

cdef inline unsigned char str_1_get(unsigned char* s, Py_ssize_t i) nogil:
LEFTazs marked this conversation as resolved.
Show resolved Hide resolved
"""
Takes an index of a 1-indexed string
and returns that character
"""
if i - 1 < 0:
return 0
return s[i - 1]

cdef inline int int_array_1_get(int[:] s, Py_ssize_t i) nogil:
"""
Takes an index of a 1-indexed int array
and returns that number
"""
if i - 1 < 0:
LEFTazs marked this conversation as resolved.
Show resolved Hide resolved
return 0
return s[i - 1]

# End helper functions
Expand Down Expand Up @@ -179,12 +196,11 @@ def damerau_levenshtein(
if transpose_costs is None:
transpose_costs = unit_matrix

s1 = str(str1).encode()
s2 = str(str2).encode()
intarr1 = convert_string_to_int_array(str1)
intarr2 = convert_string_to_int_array(str2)

return c_damerau_levenshtein(
s1, len(s1),
s2, len(s2),
intarr1, intarr2,
insert_costs,
delete_costs,
substitute_costs,
Expand All @@ -195,8 +211,7 @@ dam_lev = damerau_levenshtein


cdef DTYPE_t c_damerau_levenshtein(
unsigned char* str1, Py_ssize_t len1,
unsigned char* str2, Py_ssize_t len2,
int[:] str1, int[:] str2,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
Expand All @@ -208,11 +223,14 @@ cdef DTYPE_t c_damerau_levenshtein(
Py_ssize_t[ALPHABET_SIZE] da

Py_ssize_t i, j
unsigned char char_i, char_j
unsigned int char_i, char_j
DTYPE_t cost, ret_val
Py_ssize_t db, k, l

Array2D d
Py_ssize_t len1, len2

len1 = str1.shape[0]
LEFTazs marked this conversation as resolved.
Show resolved Hide resolved
len2 = str2.shape[0]

Array2D_init(&d, len1 + 2, len2 + 2)

Expand All @@ -230,21 +248,20 @@ cdef DTYPE_t c_damerau_levenshtein(
# fill row 0 and column 0 with insertion and deletion costs
Array2D_n1_at(d, 0, 0)[0] = 0
for i in range(1, len1 + 1):
char_i = str_1_get(str1, i)
char_i = int_array_1_get(str1, i)
cost = delete_costs[char_i]
Array2D_n1_at(d, i, 0)[0] = Array2D_n1_get(d, i - 1, 0) + cost
for j in range(1, len2 + 1):
char_j = str_1_get(str2, j)
char_j = int_array_1_get(str2, j)
cost = insert_costs[char_j]
Array2D_n1_at(d, 0, j)[0] = Array2D_n1_get(d, 0, j - 1) + cost

# fill DP array
for i in range(1, len1 + 1):
char_i = str_1_get(str1, i)

char_i = int_array_1_get(str1, i)
db = 0
for j in range(1, len2 + 1):
char_j = str_1_get(str2, j)
char_j = int_array_1_get(str2, j)

k = da[char_j]
l = db
Expand All @@ -260,7 +277,7 @@ cdef DTYPE_t c_damerau_levenshtein(
Array2D_n1_get(d, i - 1, j) + delete_costs[char_i], # delete
Array2D_n1_get(d, k - 1, l - 1) + # transpose
col_delete_range_cost(d, k + 1, i - 1) + # delete chars in between
transpose_costs[str_1_get(str1, k), str_1_get(str1, i)] + # transpose chars
transpose_costs[int_array_1_get(str1, k), int_array_1_get(str1, i)] + # transpose chars
row_insert_range_cost(d, l + 1, j - 1) # insert chars in between
)

Expand Down Expand Up @@ -307,12 +324,11 @@ def optimal_string_alignment(
if transpose_costs is None:
transpose_costs = unit_matrix

s1 = str(str1).encode()
s2 = str(str2).encode()
intarr1 = convert_string_to_int_array(str1)
intarr2 = convert_string_to_int_array(str2)

return c_optimal_string_alignment(
s1, len(s1),
s2, len(s2),
intarr1, intarr2,
insert_costs,
delete_costs,
substitute_costs,
Expand All @@ -323,8 +339,7 @@ osa = optimal_string_alignment


cdef DTYPE_t c_optimal_string_alignment(
unsigned char* str1, Py_ssize_t len1,
unsigned char* str2, Py_ssize_t len2,
int[:] str1, int[:] str2,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
Expand All @@ -334,26 +349,30 @@ cdef DTYPE_t c_optimal_string_alignment(
"""
cdef:
Py_ssize_t i, j
unsigned char char_i, char_j, prev_char_i, prev_char_j
unsigned int char_i, char_j, prev_char_i, prev_char_j
DTYPE_t ret_val
Array2D d
Py_ssize_t len1, len2

len1 = str1.shape[0]
len2 = str2.shape[0]

Array2D_init(&d, len1 + 1, len2 + 1)

# fill row 0 and column 0 with insertion and deletion costs
Array2D_0_at(d, 0, 0)[0] = 0
for i in range(1, len1 + 1):
char_i = str_1_get(str1, i)
char_i = int_array_1_get(str1, i)
Array2D_0_at(d, i, 0)[0] = Array2D_0_get(d, i - 1, 0) + delete_costs[char_i]
for j in range(1, len2 + 1):
char_j = str_1_get(str2, j)
char_j = int_array_1_get(str2, j)
Array2D_0_at(d, 0, j)[0] = Array2D_0_get(d, 0, j - 1) + insert_costs[char_j]

# fill DP array
for i in range(1, len1 + 1):
char_i = str_1_get(str1, i)
char_i = int_array_1_get(str1, i)
for j in range(1, len2 + 1):
char_j = str_1_get(str2, j)
char_j = int_array_1_get(str2, j)
if char_i == char_j: # match
Array2D_0_at(d, i, j)[0] = Array2D_0_get(d, i - 1, j - 1)
else:
Expand All @@ -364,8 +383,8 @@ cdef DTYPE_t c_optimal_string_alignment(
)

if i > 1 and j > 1:
prev_char_i = str_1_get(str1, i - 1)
prev_char_j = str_1_get(str2, j - 1)
prev_char_i = int_array_1_get(str1, i - 1)
prev_char_j = int_array_1_get(str2, j - 1)
if char_i == prev_char_j and prev_char_i == char_j: # transpose
Array2D_0_at(d, i, j)[0] = min(
Array2D_0_get(d, i, j),
Expand Down Expand Up @@ -408,12 +427,11 @@ def levenshtein(
if substitute_costs is None:
substitute_costs = unit_matrix

s1 = str(str1).encode()
s2 = str(str2).encode()
intarr1 = convert_string_to_int_array(str1)
intarr2 = convert_string_to_int_array(str2)

return c_levenshtein(
s1, len(s1),
s2, len(s2),
intarr1, intarr2,
insert_costs,
delete_costs,
substitute_costs
Expand All @@ -423,8 +441,7 @@ lev = levenshtein


cdef DTYPE_t c_levenshtein(
unsigned char* str1, Py_ssize_t len1,
unsigned char* str2, Py_ssize_t len2,
int[:] str1, int[:] str2,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs) nogil:
Expand All @@ -433,24 +450,28 @@ cdef DTYPE_t c_levenshtein(
"""
cdef:
Py_ssize_t i, j
unsigned char char_i, char_j
unsigned int char_i, char_j
DTYPE_t ret_val
Array2D d
Py_ssize_t len1, len2

len1 = str1.shape[0]
len2 = str2.shape[0]

Array2D_init(&d, len1 + 1, len2 + 1)

Array2D_0_at(d, 0, 0)[0] = 0
for i in range(1, len1 + 1):
char_i = str_1_get(str1, i)
char_i = int_array_1_get(str1, i)
Array2D_0_at(d, i, 0)[0] = Array2D_0_get(d, i - 1, 0) + delete_costs[char_i]
for j in range(1, len2 + 1):
char_j = str_1_get(str2, j)
char_j = int_array_1_get(str2, j)
Array2D_0_at(d, 0, j)[0] = Array2D_0_get(d, 0, j - 1) + insert_costs[char_j]

for i in range(1, len1 + 1):
char_i = str_1_get(str1, i)
char_i = int_array_1_get(str1, i)
for j in range(1, len2 + 1):
char_j = str_1_get(str2, j)
char_j = int_array_1_get(str2, j)
if char_i == char_j: # match
Array2D_0_at(d, i, j)[0] = Array2D_0_get(d, i - 1, j - 1)
else:
Expand Down