Add RWKV, H3, Hyena

config/models/h3.yaml

@@ -0,0 +1,7 @@
+_name_: h3
+d_state: 64
+head_dim: 1
+mode: diag
+measure: diag-lin
+# lr: ${eval:"min(0.001, ${optimizer.lr})"}
+lr: 0.001

config/models/hyena.yaml

@@ -0,0 +1,16 @@
+_name_: hyena
+l_max: 1024
+order: 2
+filter_order: 64
+num_heads: 1
+inner_factor: 1
+num_blocks: 1
+fused_bias_fc: false
+outer_mixing: false
+dropout: 0.0 
+filter_dropout: 0.0
+filter_cls: 'hyena-filter'
+post_order_ffn: false
+jit_filter: false
+short_filter_order: 3
+activation: "id"

config/models/lstm.yaml

@@ -0,0 +1 @@
+_name_: LSTM_Transformer