Add RWKV, H3, Hyena

2023-08-05 17:33:32 +02:00
parent a71030547c
commit 7b15a413d4
22 changed files with 1794 additions and 0 deletions
@@ -0,0 +1,7 @@
+_name_: h3
+d_state: 64
+head_dim: 1
+mode: diag
+measure: diag-lin
+# lr: ${eval:"min(0.001, ${optimizer.lr})"}
+lr: 0.001
@@ -0,0 +1,16 @@
+_name_: hyena
+l_max: 1024
+order: 2
+filter_order: 64
+num_heads: 1
+inner_factor: 1
+num_blocks: 1
+fused_bias_fc: false
+outer_mixing: false
+dropout: 0.0 
+filter_dropout: 0.0
+filter_cls: 'hyena-filter'
+post_order_ffn: false
+jit_filter: false
+short_filter_order: 3
+activation: "id"
@@ -0,0 +1 @@
+_name_: LSTM_Transformer