1# Version 🅰️
2huggingface-cli download YOUR_USERNAME/flash-attn-windows-blackwell \
3 flash_attn-2.8.4+cu13torch2.13cxx11abiTRUE-cp313-cp313-win_amd64.whl \
4 --local-dir ./
5
6# Version 🅱️
7huggingface-cli download YOUR_USERNAME/flash-attn-windows-blackwell \
8 flash_attn-2.8.3+cu13torch2.9.1cxx11abiTRUE-cp312-cp312-win_amd64.whl \
9 --local-dir ./
1from huggingface_hub import hf_hub_download
2
3path = hf_hub_download(
4 repo_id="YOUR_USERNAME/flash-attn-windows-blackwell",
5 filename="flash_attn-2.8.4+cu13torch2.13cxx11abiTRUE-cp313-cp313-win_amd64.whl"
6)
7print(path)
1import torch
2from flash_attn import flash_attn_func
3import flash_attn
4
5def test_flash_attn():
6 print(f"flash_attn version: {flash_attn.__version__}")
7
8 if not torch.cuda.is_available():
9 print("❌ CUDA is not available. Flash Attention requires GPU support.")
10 return False
11
12 # Small dimensions to avoid OOM
13 batch_size, seq_len, n_heads, head_dim = 2, 8, 4, 32
14 dtype = torch.float16
15 device = "cuda"
16
17 # Random q, k, v with gradients
18 q = torch.randn((batch_size, seq_len, n_heads, head_dim), dtype=dtype, device=device, requires_grad=True)
19 k = torch.randn((batch_size, seq_len, n_heads, head_dim), dtype=dtype, device=device, requires_grad=True)
20 v = torch.randn((batch_size, seq_len, n_heads, head_dim), dtype=dtype, device=device, requires_grad=True)
21
22 # Forward pass
23 out = flash_attn_func(q, k, v, causal=False)
24
25 # Check output shape
26 expected = (batch_size, seq_len, n_heads, head_dim)
27 if out.shape != expected:
28 print(f"❌ Output shape mismatch: {out.shape} != {expected}")
29 return False
30
31 # Backward pass (test gradients)
32 loss = out.sum()
33 loss.backward()
34
35 if q.grad is None or k.grad is None or v.grad is None:
36 print("❌ Gradient computation failed")
37 return False
38
39 print("✅ Flash Attention test passed!")
40 return True
41
42if __name__ == "__main__":
43 test_flash_attn()
This is a compiled artifact of
flash-attention, licensed under the
BSD 3-Clause "New" or "Revised" License.
