Skip to content

Commit

Permalink
need for another project
Browse files Browse the repository at this point in the history
  • Loading branch information
lucidrains committed Sep 20, 2024
1 parent 9d8bd8a commit 30822a8
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 2 deletions.
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
setup(
name = 'x-transformers',
packages = find_packages(exclude=['examples']),
version = '1.35.2',
version = '1.35.3',
license='MIT',
description = 'X-Transformers - Pytorch',
author = 'Phil Wang',
Expand Down
7 changes: 6 additions & 1 deletion x_transformers/x_transformers.py
Original file line number Diff line number Diff line change
Expand Up @@ -920,6 +920,7 @@ def __init__(
kv_heads = None,
shared_kv = False,
value_dim_head = None,
dim_out = None,
tensor_product = False, # https://arxiv.org/abs/2208.06061
add_zero_kv = False, # same as add_zero_attn in pytorch
rotary_embed_values = False,
Expand Down Expand Up @@ -1057,7 +1058,11 @@ def __init__(
# attention on attention

self.attn_on_attn = on_attn
self.to_out = nn.Sequential(nn.Linear(out_dim, dim * 2, bias = False), nn.GLU()) if on_attn else nn.Linear(out_dim, dim, bias = False)

# output dimension by default same as input, but can be overridden

dim_out = default(dim_out, dim)
self.to_out = nn.Sequential(nn.Linear(out_dim, dim_out * 2, bias = False), nn.GLU()) if on_attn else nn.Linear(out_dim, dim_out, bias = False)

# whether to rotate positions into values, for absolute positions in addition to relative

Expand Down

0 comments on commit 30822a8

Please sign in to comment.