File tree Expand file tree Collapse file tree 3 files changed +5
-2
lines changed
server/text_generation_server Expand file tree Collapse file tree 3 files changed +5
-2
lines changed Original file line number Diff line number Diff line change @@ -149,7 +149,7 @@ def __init__(
149
149
):
150
150
super ().__init__ ()
151
151
self .max_past = (
152
- config .sliding_window if config .sliding_window is not None else 0
152
+ config .sliding_window if config .sliding_window is not None else - 1
153
153
)
154
154
self .num_heads = config .num_attention_heads
155
155
self .hidden_size = config .hidden_size
Original file line number Diff line number Diff line change @@ -204,7 +204,7 @@ def __init__(
204
204
):
205
205
super ().__init__ ()
206
206
self .max_past = (
207
- config .sliding_window if config .sliding_window is not None else 0
207
+ config .sliding_window if config .sliding_window is not None else - 1
208
208
)
209
209
self .num_heads = config .num_attention_heads
210
210
self .hidden_size = config .hidden_size
Original file line number Diff line number Diff line change @@ -72,6 +72,9 @@ def attention(
72
72
softmax_scale ,
73
73
window_size_left = - 1 ,
74
74
):
75
+ if window_size_left <= 0 and window_size_left != - 1 :
76
+ raise ValueError ("`window_size_left` must be > 0 or -1" )
77
+
75
78
if HAS_FLASH_ATTN_V2_CUDA :
76
79
return flash_attn_2_cuda .varlen_fwd (
77
80
q ,
You can’t perform that action at this time.
0 commit comments