From a71030547c096ca6bb7ff85140b27e86a3cbcab4 Mon Sep 17 00:00:00 2001 From: martin Date: Sat, 5 Aug 2023 15:05:06 +0200 Subject: [PATCH] hyena predict longer sequences --- .../simple_hyena_model-checkpoint.ipynb | 233 +++-- ...ena_predict_more_than_one-checkpoint.ipynb | 850 ++++++++++++++++++ hyena_test/simple_hyena_model.ipynb | 121 ++- .../simple_hyena_predict_more_than_one.ipynb | 850 ++++++++++++++++++ 4 files changed, 1981 insertions(+), 73 deletions(-) create mode 100644 hyena_test/.ipynb_checkpoints/simple_hyena_predict_more_than_one-checkpoint.ipynb create mode 100644 hyena_test/simple_hyena_predict_more_than_one.ipynb diff --git a/hyena_test/.ipynb_checkpoints/simple_hyena_model-checkpoint.ipynb b/hyena_test/.ipynb_checkpoints/simple_hyena_model-checkpoint.ipynb index 2ef8861..f630e95 100644 --- a/hyena_test/.ipynb_checkpoints/simple_hyena_model-checkpoint.ipynb +++ b/hyena_test/.ipynb_checkpoints/simple_hyena_model-checkpoint.ipynb @@ -12,7 +12,7 @@ "text": [ "torch.Size([1, 1024, 128]) torch.Size([1, 1024, 128])\n", "Causality check: gradients should not flow \"from future to past\"\n", - "tensor(2.1330e-09) tensor(0.2463)\n" + "tensor(-4.1268e-09) tensor(0.0844)\n" ] } ], @@ -291,7 +291,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "id": "032ef08a-8cc6-491a-9eb8-4a6b3f2d165e", "metadata": {}, "outputs": [ @@ -343,7 +343,9 @@ " # Increase the channel dimension from 1 to d_model\n", " u = self.fc_before(u) \n", " # Pass through the operator\n", + " #[B,1024,128] --> [B,1024,128]\n", " u = self.operator(u)\n", + " \n", " last_state = u[:,-1,:]\n", " # Decrease the channel dimension back to 1\n", " y = self.fc_after(last_state)\n", @@ -371,7 +373,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 3, "id": "80cde67b-992f-4cb0-8824-4a6b7e4984ca", "metadata": {}, "outputs": [ @@ -379,16 +381,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "Train Epoch: 1 [0/640 (0%)]\tLoss: 0.736030\n", - "Train Epoch: 2 [0/640 (0%)]\tLoss: 0.013385\n", - "Train Epoch: 3 [0/640 (0%)]\tLoss: 0.019001\n", - "Train Epoch: 4 [0/640 (0%)]\tLoss: 0.010262\n", - "Train Epoch: 5 [0/640 (0%)]\tLoss: 0.005347\n", - "Train Epoch: 6 [0/640 (0%)]\tLoss: 0.006345\n", - "Train Epoch: 7 [0/640 (0%)]\tLoss: 0.004454\n", - "Train Epoch: 8 [0/640 (0%)]\tLoss: 0.003857\n", - "Train Epoch: 9 [0/640 (0%)]\tLoss: 0.003062\n", - "Train Epoch: 10 [0/640 (0%)]\tLoss: 0.002607\n" + "Train Epoch: 1 [0/640 (0%)]\tLoss: 0.446847\n", + "Train Epoch: 2 [0/640 (0%)]\tLoss: 0.077979\n", + "Train Epoch: 3 [0/640 (0%)]\tLoss: 0.021656\n", + "Train Epoch: 4 [0/640 (0%)]\tLoss: 0.007355\n", + "Train Epoch: 5 [0/640 (0%)]\tLoss: 0.004926\n", + "Train Epoch: 6 [0/640 (0%)]\tLoss: 0.006014\n", + "Train Epoch: 7 [0/640 (0%)]\tLoss: 0.003400\n", + "Train Epoch: 8 [0/640 (0%)]\tLoss: 0.003720\n", + "Train Epoch: 9 [0/640 (0%)]\tLoss: 0.004267\n", + "Train Epoch: 10 [0/640 (0%)]\tLoss: 0.004081\n" ] } ], @@ -495,49 +497,155 @@ }, { "cell_type": "code", - "execution_count": 17, - "id": "1b763e03-baab-4b02-bae0-5747461bca7f", + "execution_count": 4, + "id": "90330622-8b44-4b45-8158-6840538f768c", "metadata": {}, "outputs": [], "source": [ - "def correlate(model, device, data_loader):\n", - " model.eval()\n", - " correlations = {key: [] for key in [\"frequency\", \"phase\", \"amplitude\", \"noise_sd\"]}\n", - " with torch.no_grad():\n", - " for data, target, params in data_loader:\n", + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.metrics import r2_score\n", "\n", - " data, target = data.to(device), target.to(device)\n", - " output, last_state = model(data)\n", - " last_state_np = last_state.cpu().numpy()\n", - " params_np = params.cpu().numpy()\n", - " #import pdb;pdb.set_trace()\n", - "\n", - " # Compute correlations between last_state and parameters\n", - " for i, key in enumerate(correlations.keys()):\n", - " correlation = np.corrcoef(last_state_np.squeeze(), params_np[:,i])[0,1]\n", - " correlations[key].append(correlation)\n", + "def fit_and_evaluate_linear_regression(outputs_and_params):\n", + " # Split the data into inputs (last_states) and targets (params)\n", + " inputs = np.concatenate([x[0] for x in outputs_and_params])\n", + " targets = np.concatenate([x[1] for x in outputs_and_params])\n", " \n", - " # Average correlations over all batches\n", - " avg_correlations = {key: np.mean(value) for key, value in correlations.items()}\n", - " return avg_correlations" + " r2_scores = []\n", + " param_names = [\"frequency\", \"phase\", \"amplitude\", \"noise_sd\"]\n", + " \n", + " # Fit the linear regression model for each parameter and calculate the R^2 score\n", + " for i in range(targets.shape[1]):\n", + " model = LinearRegression().fit(inputs, targets[:, i])\n", + " pred = model.predict(inputs)\n", + " score = r2_score(targets[:, i], pred)\n", + " r2_scores.append(score)\n", + " print(f\"R^2 score for {param_names[i]}: {score:.2f}\")\n", + " \n", + " return r2_scores" ] }, { "cell_type": "code", - "execution_count": 18, - "id": "f4c78c51-a538-4d24-ab7b-fb6035c78df8", + "execution_count": 5, + "id": "5eb62a22-cad8-43c4-b757-f36b6a01e9be", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_outputs(model, device, data_loader):\n", + " model.eval()\n", + " outputs_and_params = []\n", + " with torch.no_grad():\n", + " for data, target, params in data_loader:\n", + " data, target = data.to(device), target.to(device)\n", + " output, last_state = model(data)\n", + " outputs_and_params.append((last_state.cpu().numpy(), params.cpu().numpy()))\n", + " return outputs_and_params\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a95ee542-1c39-4f04-9184-e26c6983a018", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "> \u001b[0;32m/tmp/ipykernel_9454/3342195123.py\u001b[0m(14)\u001b[0;36mcorrelate\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32m 12 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0m\u001b[0;32m 13 \u001b[0;31m \u001b[0;31m# Compute correlations between last_state and parameters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0m\u001b[0;32m---> 14 \u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorrelations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0m\u001b[0;32m 15 \u001b[0;31m \u001b[0mcorrelation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcorrcoef\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlast_state_np\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams_np\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0m\u001b[0;32m 16 \u001b[0;31m \u001b[0mcorrelations\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorrelation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "R^2 score for frequency: 0.77\n", + "R^2 score for phase: 0.66\n", + "R^2 score for amplitude: 0.99\n", + "R^2 score for noise_sd: 0.97\n" + ] + } + ], + "source": [ + "outputs_and_params = generate_outputs(model, device, train_loader)\n", + "\n", + "# Fit the linear regression model and print the R^2 score for each parameter\n", + "r2_scores = fit_and_evaluate_linear_regression(outputs_and_params)" + ] + }, + { + "cell_type": "markdown", + "id": "de65d0d2-b0c6-4ac5-a87f-70fa7f90480b", + "metadata": {}, + "source": [ + "# Autoregressive" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "8ee139a6-aee5-4309-8685-bf0c28893279", + "metadata": {}, + "outputs": [], + "source": [ + "def predict_autoregressive(model, initial_data, n_steps):\n", + " model.eval()\n", + " predictions = []\n", + " current_input = initial_data\n", + " with torch.no_grad():\n", + " for _ in range(n_steps):\n", + " # Get the prediction for the next step and save it\n", + " import pdb;pdb.set_trace()\n", + " next_output, _ = model(current_input)\n", + " predictions.append(next_output)\n", + "\n", + " # Prepare the input for the next step\n", + " next_input = torch.cat((current_input[:, 1:, :], next_output[:, -1:, :]), dim=1)\n", + "\n", + " current_input = next_input\n", + "\n", + " return torch.cat(predictions, dim=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "272040d6-4dfb-438b-a432-744e950effd1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "9f49cbf8-08e8-428a-af80-bcb2515a6327", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 1024, 1])" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "initial_data = dataset[0]\n", + "initial_data[0][None,...].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "c75464e3-e44d-4325-8804-29d8081e3a45", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/tmp/ipykernel_9626/3477884695.py\u001b[0m(9)\u001b[0;36mpredict_autoregressive\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 7 \u001b[0;31m \u001b[0;31m# Get the prediction for the next step and save it\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 8 \u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0mpdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m----> 9 \u001b[0;31m \u001b[0mnext_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcurrent_input\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 10 \u001b[0;31m \u001b[0mpredictions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 11 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0m\n" ] }, @@ -545,53 +653,34 @@ "name": "stdin", "output_type": "stream", "text": [ - "ipdb> last_state_np.shape\n" + "ipdb> current_input\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "(64, 128)\n" + "tensor([[-1.2113],\n", + " [-1.2368],\n", + " [-1.1851],\n", + " ...,\n", + " [-1.1084],\n", + " [-0.9648],\n", + " [-1.0586]])\n" ] }, { "name": "stdin", "output_type": "stream", "text": [ - "ipdb> last_state_np[0]\n" + "ipdb> current_input.shape\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "array([ 0.13440676, -0.03606963, -0.4140934 , 0.5431792 , 0.36556095,\n", - " 0.2256596 , -0.4616309 , -0.05567896, 0.17625177, -0.23529659,\n", - " -0.5208519 , 0.29691923, 0.15615058, 0.31342992, -0.5054718 ,\n", - " -0.33130994, -0.03956199, 0.31403548, -0.15925817, -0.22006416,\n", - " 0.00838468, -0.30691615, -0.1828884 , -0.52498204, 0.07198659,\n", - " 0.38572663, -0.27560705, 0.12110637, -0.17199083, 0.3913066 ,\n", - " -0.03978934, -0.21167544, -0.43025637, 0.20562531, 0.3000516 ,\n", - " -0.6784174 , -0.04233613, 0.4706083 , 0.20292807, 0.49932548,\n", - " 0.00203749, 0.2665777 , -0.16989222, 0.40648764, 0.22203793,\n", - " -0.44289762, 0.20751204, -0.38801843, -0.001487 , -0.49365598,\n", - " 0.05991718, -0.10120638, 0.36523518, -0.15450253, 0.11142011,\n", - " -0.20295474, 0.12229299, 0.09449576, -0.3422598 , 0.18969077,\n", - " 0.517254 , 0.08046471, 0.02134303, -0.35802346, -0.26192123,\n", - " 0.26145002, 0.11439252, 0.03314593, -0.15331428, 0.42282102,\n", - " 0.6026961 , -0.04233361, -0.5652172 , 0.33544067, 0.05744396,\n", - " 0.43544483, 0.2176097 , 0.22265801, -0.03894311, -0.01405966,\n", - " 0.23479447, 0.32931918, 0.21597862, 0.40402904, 0.20630498,\n", - " 0.09036086, -0.16922598, -0.1774486 , -0.14753146, -0.22214624,\n", - " -0.19101782, -0.09274255, 0.10928088, -0.01354241, -0.3864469 ,\n", - " 0.46331462, -0.38134843, -0.07766411, 0.750954 , -0.06306303,\n", - " -0.33691666, -0.1798551 , 0.19826202, -0.13544285, 0.01956506,\n", - " 0.6431204 , -0.11272874, 0.1345196 , 0.23029736, 0.28865197,\n", - " 0.70087713, 0.3592593 , 0.30329305, -0.26943353, -0.11942452,\n", - " -0.21187985, 0.19452253, 0.05659255, -0.00958484, -0.33417243,\n", - " -0.14836329, 0.28580692, 0.20885246, 0.18010336, 0.56253076,\n", - " -0.25303417, 0.0189368 , 0.2504725 ], dtype=float32)\n" + "torch.Size([1024, 1])\n" ] }, { @@ -603,13 +692,13 @@ } ], "source": [ - "correlate(model,\"cpu\",train_loader)" + "predict_autoregressive(model,initial_data[0][None,...],100)" ] }, { "cell_type": "code", "execution_count": null, - "id": "e1558ffb-4699-4a8c-b05d-c3ac31a3829f", + "id": "90a37c56-59f3-49bc-bfbe-d9e126c42ed1", "metadata": {}, "outputs": [], "source": [] diff --git a/hyena_test/.ipynb_checkpoints/simple_hyena_predict_more_than_one-checkpoint.ipynb b/hyena_test/.ipynb_checkpoints/simple_hyena_predict_more_than_one-checkpoint.ipynb new file mode 100644 index 0000000..f4f18d1 --- /dev/null +++ b/hyena_test/.ipynb_checkpoints/simple_hyena_predict_more_than_one-checkpoint.ipynb @@ -0,0 +1,850 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 32, + "id": "6c6e33cb-72f9-42fa-936a-33b5fe338d15", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 1024, 128]) torch.Size([1, 1024, 128])\n", + "Causality check: gradients should not flow \"from future to past\"\n", + "tensor(-6.7998e-10) tensor(0.1017)\n" + ] + } + ], + "source": [ + "# %load standalone_hyena.py\n", + "\"\"\"\n", + "Simplified standalone version of Hyena: https://arxiv.org/abs/2302.10866, designed for quick experimentation.\n", + "A complete version is available under `src.models.sequence.hyena`.\n", + "\"\"\"\n", + "\n", + "import math\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from einops import rearrange\n", + "\n", + "\n", + "def fftconv(u, k, D):\n", + " seqlen = u.shape[-1]\n", + " fft_size = 2 * seqlen\n", + " \n", + " k_f = torch.fft.rfft(k, n=fft_size) / fft_size\n", + " u_f = torch.fft.rfft(u.to(dtype=k.dtype), n=fft_size)\n", + " \n", + " if len(u.shape) > 3: k_f = k_f.unsqueeze(1)\n", + " y = torch.fft.irfft(u_f * k_f, n=fft_size, norm='forward')[..., :seqlen]\n", + "\n", + " out = y + u * D.unsqueeze(-1)\n", + " return out.to(dtype=u.dtype)\n", + "\n", + "\n", + "@torch.jit.script \n", + "def mul_sum(q, y):\n", + " return (q * y).sum(dim=1)\n", + "\n", + "class OptimModule(nn.Module):\n", + " \"\"\" Interface for Module that allows registering buffers/parameters with configurable optimizer hyperparameters \"\"\"\n", + "\n", + " def register(self, name, tensor, lr=None, wd=0.0):\n", + " \"\"\"Register a tensor with a configurable learning rate and 0 weight decay\"\"\"\n", + "\n", + " if lr == 0.0:\n", + " self.register_buffer(name, tensor)\n", + " else:\n", + " self.register_parameter(name, nn.Parameter(tensor))\n", + "\n", + " optim = {}\n", + " if lr is not None: optim[\"lr\"] = lr\n", + " if wd is not None: optim[\"weight_decay\"] = wd\n", + " setattr(getattr(self, name), \"_optim\", optim)\n", + " \n", + "\n", + "class Sin(nn.Module):\n", + " def __init__(self, dim, w=10, train_freq=True):\n", + " super().__init__()\n", + " self.freq = nn.Parameter(w * torch.ones(1, dim)) if train_freq else w * torch.ones(1, dim)\n", + "\n", + " def forward(self, x):\n", + " return torch.sin(self.freq * x)\n", + " \n", + " \n", + "class PositionalEmbedding(OptimModule):\n", + " def __init__(self, emb_dim: int, seq_len: int, lr_pos_emb: float=1e-5, **kwargs): \n", + " \"\"\"Complex exponential positional embeddings for Hyena filters.\"\"\" \n", + " super().__init__()\n", + " \n", + " self.seq_len = seq_len\n", + " # The time embedding fed to the filteres is normalized so that t_f = 1\n", + " t = torch.linspace(0, 1, self.seq_len)[None, :, None] # 1, L, 1\n", + " \n", + " if emb_dim > 1:\n", + " bands = (emb_dim - 1) // 2 \n", + " # To compute the right embeddings we use the \"proper\" linspace \n", + " t_rescaled = torch.linspace(0, seq_len - 1, seq_len)[None, :, None]\n", + " w = 2 * math.pi * t_rescaled / seq_len # 1, L, 1 \n", + " \n", + " f = torch.linspace(1e-4, bands - 1, bands)[None, None] \n", + " z = torch.exp(-1j * f * w)\n", + " z = torch.cat([t, z.real, z.imag], dim=-1)\n", + " self.register(\"z\", z, lr=lr_pos_emb) \n", + " self.register(\"t\", t, lr=0.0)\n", + " \n", + " def forward(self, L):\n", + " return self.z[:, :L], self.t[:, :L]\n", + " \n", + "\n", + "class ExponentialModulation(OptimModule):\n", + " def __init__(\n", + " self,\n", + " d_model,\n", + " fast_decay_pct=0.3,\n", + " slow_decay_pct=1.5,\n", + " target=1e-2,\n", + " modulation_lr=0.0,\n", + " modulate: bool=True,\n", + " shift: float = 0.0,\n", + " **kwargs\n", + " ):\n", + " super().__init__()\n", + " self.modulate = modulate\n", + " self.shift = shift\n", + " max_decay = math.log(target) / fast_decay_pct\n", + " min_decay = math.log(target) / slow_decay_pct\n", + " deltas = torch.linspace(min_decay, max_decay, d_model)[None, None]\n", + " self.register(\"deltas\", deltas, lr=modulation_lr)\n", + " \n", + " def forward(self, t, x):\n", + " if self.modulate:\n", + " decay = torch.exp(-t * self.deltas.abs()) \n", + " x = x * (decay + self.shift)\n", + " return x \n", + "\n", + "\n", + "class HyenaFilter(OptimModule):\n", + " def __init__(\n", + " self, \n", + " d_model,\n", + " emb_dim=3, # dim of input to MLP, augments with positional encoding\n", + " order=16, # width of the implicit MLP \n", + " fused_fft_conv=False,\n", + " seq_len=1024, \n", + " lr=1e-3, \n", + " lr_pos_emb=1e-5,\n", + " dropout=0.0, \n", + " w=1, # frequency of periodic activations \n", + " wd=0, # weight decay of kernel parameters \n", + " bias=True,\n", + " num_inner_mlps=2,\n", + " normalized=False,\n", + " **kwargs\n", + " ):\n", + " \"\"\"\n", + " Implicit long filter with modulation.\n", + " \n", + " Args:\n", + " d_model: number of channels in the input\n", + " emb_dim: dimension of the positional encoding (`emb_dim` - 1) // 2 is the number of bands\n", + " order: width of the FFN\n", + " num_inner_mlps: number of inner linear layers inside filter MLP\n", + " \"\"\"\n", + " super().__init__()\n", + " self.d_model = d_model\n", + " self.use_bias = bias\n", + " self.fused_fft_conv = fused_fft_conv\n", + " self.bias = nn.Parameter(torch.randn(self.d_model))\n", + " self.dropout = nn.Dropout(dropout)\n", + " \n", + " act = Sin(dim=order, w=w)\n", + " self.emb_dim = emb_dim\n", + " assert emb_dim % 2 != 0 and emb_dim >= 3, \"emb_dim must be odd and greater or equal to 3 (time, sine and cosine)\"\n", + " self.seq_len = seq_len\n", + " \n", + " self.pos_emb = PositionalEmbedding(emb_dim, seq_len, lr_pos_emb)\n", + " \n", + " self.implicit_filter = nn.Sequential(\n", + " nn.Linear(emb_dim, order),\n", + " act,\n", + " )\n", + " for i in range(num_inner_mlps):\n", + " self.implicit_filter.append(nn.Linear(order, order))\n", + " self.implicit_filter.append(act)\n", + "\n", + " self.implicit_filter.append(nn.Linear(order, d_model, bias=False))\n", + " \n", + " self.modulation = ExponentialModulation(d_model, **kwargs)\n", + " \n", + " self.normalized = normalized\n", + " for c in self.implicit_filter.children():\n", + " for name, v in c.state_dict().items(): \n", + " optim = {\"weight_decay\": wd, \"lr\": lr}\n", + " setattr(getattr(c, name), \"_optim\", optim)\n", + "\n", + " def filter(self, L, *args, **kwargs):\n", + " z, t = self.pos_emb(L)\n", + " h = self.implicit_filter(z)\n", + " h = self.modulation(t, h)\n", + " return h\n", + "\n", + " def forward(self, x, L, k=None, bias=None, *args, **kwargs):\n", + " if k is None: k = self.filter(L)\n", + " \n", + " # Ensure compatibility with filters that return a tuple \n", + " k = k[0] if type(k) is tuple else k \n", + "\n", + " y = fftconv(x, k, bias)\n", + " return y\n", + " \n", + " \n", + "class HyenaOperator(nn.Module):\n", + " def __init__(\n", + " self,\n", + " d_model,\n", + " l_max,\n", + " order=2, \n", + " filter_order=64,\n", + " dropout=0.0, \n", + " filter_dropout=0.0, \n", + " **filter_args,\n", + " ):\n", + " r\"\"\"\n", + " Hyena operator described in the paper https://arxiv.org/pdf/2302.10866.pdf\n", + " \n", + " Args:\n", + " d_model (int): Dimension of the input and output embeddings (width of the layer)\n", + " l_max: (int): Maximum input sequence length. Defaults to None\n", + " order: (int): Depth of the Hyena recurrence. Defaults to 2\n", + " dropout: (float): Dropout probability. Defaults to 0.0\n", + " filter_dropout: (float): Dropout probability for the filter. Defaults to 0.0\n", + " \"\"\"\n", + " super().__init__()\n", + " self.d_model = d_model\n", + " self.l_max = l_max\n", + " self.order = order\n", + " inner_width = d_model * (order + 1)\n", + " self.dropout = nn.Dropout(dropout)\n", + " self.in_proj = nn.Linear(d_model, inner_width)\n", + " self.out_proj = nn.Linear(d_model, d_model)\n", + " \n", + " self.short_filter = nn.Conv1d(\n", + " inner_width, \n", + " inner_width, \n", + " 3,\n", + " padding=2,\n", + " groups=inner_width\n", + " )\n", + " self.filter_fn = HyenaFilter(\n", + " d_model * (order - 1), \n", + " order=filter_order, \n", + " seq_len=l_max,\n", + " channels=1, \n", + " dropout=filter_dropout, \n", + " **filter_args\n", + " ) \n", + "\n", + " def forward(self, u, *args, **kwargs):\n", + " l = u.size(-2)\n", + " l_filter = min(l, self.l_max)\n", + " u = self.in_proj(u)\n", + " u = rearrange(u, 'b l d -> b d l')\n", + " \n", + " uc = self.short_filter(u)[...,:l_filter] \n", + " *x, v = uc.split(self.d_model, dim=1)\n", + " \n", + " k = self.filter_fn.filter(l_filter)[0]\n", + " k = rearrange(k, 'l (o d) -> o d l', o=self.order - 1)\n", + " bias = rearrange(self.filter_fn.bias, '(o d) -> o d', o=self.order - 1)\n", + " \n", + " for o, x_i in enumerate(reversed(x[1:])):\n", + " v = self.dropout(v * x_i)\n", + " v = self.filter_fn(v, l_filter, k=k[o], bias=bias[o])\n", + "\n", + " y = rearrange(v * x[0], 'b d l -> b l d')\n", + "\n", + " y = self.out_proj(y)\n", + " return y\n", + "\n", + " \n", + " \n", + "if __name__ == \"__main__\":\n", + " layer = HyenaOperator(\n", + " \n", + " d_model=128, \n", + " l_max=1024, \n", + " order=2, \n", + " filter_order=64\n", + " )\n", + " x = torch.randn(1, 1024, 128, requires_grad=True)\n", + " y = layer(x)\n", + " \n", + " print(x.shape, y.shape)\n", + " \n", + " grad = torch.autograd.grad(y[:, 10, :].sum(), x)[0]\n", + " print('Causality check: gradients should not flow \"from future to past\"')\n", + " print(grad[0, 11, :].sum(), grad[0, 9, :].sum())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 368, + "id": "032ef08a-8cc6-491a-9eb8-4a6b3f2d165e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 1023, 1]) torch.Size([1, 500])\n" + ] + } + ], + "source": [ + "class HyenaOperatorAutoregressive1D(nn.Module):\n", + " def __init__(\n", + " self,\n", + " d_model,\n", + " l_max,\n", + " order=2, \n", + " filter_order=64,\n", + " dropout=0.0, \n", + " filter_dropout=0.0, \n", + " **filter_args,\n", + " ):\n", + " super().__init__()\n", + "\n", + " self.l_max = l_max\n", + " self.d_model = d_model\n", + " self.l_max = l_max\n", + " self.order = order\n", + " inner_width = d_model * (order + 1)\n", + "\n", + " self.dropout = nn.Dropout(dropout)\n", + " self.in_proj = nn.Linear(d_model, inner_width)\n", + " self.out_proj = nn.Linear(d_model, d_model)\n", + " self.fc_before = nn.Linear(1, d_model) # Fully connected layer before the main layer\n", + " self.fc_after = nn.Linear(d_model, 500) # Fully connected layer after the main layer\n", + "\n", + " self.operator = HyenaOperator(\n", + " d_model=d_model,\n", + " l_max=l_max,\n", + " order=order, \n", + " filter_order=filter_order,\n", + " dropout=dropout, \n", + " filter_dropout=filter_dropout, \n", + " **filter_args,\n", + " )\n", + "\n", + " def forward(self, u, *args, **kwargs):\n", + " # Increase the channel dimension from 1 to d_model\n", + " u = self.fc_before(u) \n", + " # Pass through the operator\n", + " #[B,1024,128] --> [B,1024,128]\n", + " u = self.operator(u)\n", + " \n", + " last_state = u[:,-1,:]\n", + " # Decrease the channel dimension back to 1\n", + " y = self.fc_after(last_state)\n", + " return y,last_state\n", + "\n", + "\n", + "if __name__ == \"__main__\":\n", + " layer = HyenaOperatorAutoregressive1D(\n", + " d_model=128, \n", + " l_max=1024, \n", + " order=2, \n", + " filter_order=64\n", + " )\n", + "\n", + " x = torch.randn(1, 1023, 1, requires_grad=True) # 1D time series input\n", + " y, last_state = layer(x)\n", + "\n", + " #import pdb;pdb.set_trace()\n", + " print(x.shape, y.shape) # should now be [1, 1024, 1]\n", + "\n", + " #grad = torch.autograd.grad(y[:, 10, 0].sum(), x)[0]\n", + " #print('Causality check: gradients should not flow \"from future to past\"')\n", + " #print(grad[0, 11, 0].sum(), grad[0, 9, 0].sum())" + ] + }, + { + "cell_type": "code", + "execution_count": 369, + "id": "80cde67b-992f-4cb0-8824-4a6b7e4984ca", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train Epoch: 1 [0/640 (0%)]\tLoss: 0.569402\n", + "Train Epoch: 2 [0/640 (0%)]\tLoss: 0.459675\n", + "Train Epoch: 3 [0/640 (0%)]\tLoss: 0.455738\n", + "Train Epoch: 4 [0/640 (0%)]\tLoss: 0.334769\n", + "Train Epoch: 5 [0/640 (0%)]\tLoss: 0.400367\n", + "Train Epoch: 6 [0/640 (0%)]\tLoss: 0.353075\n", + "Train Epoch: 7 [0/640 (0%)]\tLoss: 0.372000\n", + "Train Epoch: 8 [0/640 (0%)]\tLoss: 0.354896\n", + "Train Epoch: 9 [0/640 (0%)]\tLoss: 0.370209\n", + "Train Epoch: 10 [0/640 (0%)]\tLoss: 0.323172\n", + "Train Epoch: 11 [0/640 (0%)]\tLoss: 0.324925\n", + "Train Epoch: 12 [0/640 (0%)]\tLoss: 0.319851\n", + "Train Epoch: 13 [0/640 (0%)]\tLoss: 0.267247\n", + "Train Epoch: 14 [0/640 (0%)]\tLoss: 0.307432\n", + "Train Epoch: 15 [0/640 (0%)]\tLoss: 0.270438\n", + "Train Epoch: 16 [0/640 (0%)]\tLoss: 0.274847\n", + "Train Epoch: 17 [0/640 (0%)]\tLoss: 0.283570\n", + "Train Epoch: 18 [0/640 (0%)]\tLoss: 0.282435\n", + "Train Epoch: 19 [0/640 (0%)]\tLoss: 0.325277\n", + "Train Epoch: 20 [0/640 (0%)]\tLoss: 0.326561\n", + "Train Epoch: 21 [0/640 (0%)]\tLoss: 0.305762\n", + "Train Epoch: 22 [0/640 (0%)]\tLoss: 0.298317\n", + "Train Epoch: 23 [0/640 (0%)]\tLoss: 0.277244\n", + "Train Epoch: 24 [0/640 (0%)]\tLoss: 0.271200\n", + "Train Epoch: 25 [0/640 (0%)]\tLoss: 0.246922\n", + "Train Epoch: 26 [0/640 (0%)]\tLoss: 0.231166\n", + "Train Epoch: 27 [0/640 (0%)]\tLoss: 0.301950\n", + "Train Epoch: 28 [0/640 (0%)]\tLoss: 0.272427\n", + "Train Epoch: 29 [0/640 (0%)]\tLoss: 0.278342\n", + "Train Epoch: 30 [0/640 (0%)]\tLoss: 0.246955\n", + "Train Epoch: 31 [0/640 (0%)]\tLoss: 0.304109\n", + "Train Epoch: 32 [0/640 (0%)]\tLoss: 0.254574\n", + "Train Epoch: 33 [0/640 (0%)]\tLoss: 0.234942\n", + "Train Epoch: 34 [0/640 (0%)]\tLoss: 0.269669\n", + "Train Epoch: 35 [0/640 (0%)]\tLoss: 0.239477\n", + "Train Epoch: 36 [0/640 (0%)]\tLoss: 0.282838\n", + "Train Epoch: 37 [0/640 (0%)]\tLoss: 0.264821\n", + "Train Epoch: 38 [0/640 (0%)]\tLoss: 0.236031\n", + "Train Epoch: 39 [0/640 (0%)]\tLoss: 0.239743\n", + "Train Epoch: 40 [0/640 (0%)]\tLoss: 0.289715\n", + "Train Epoch: 41 [0/640 (0%)]\tLoss: 0.240897\n", + "Train Epoch: 42 [0/640 (0%)]\tLoss: 0.212328\n", + "Train Epoch: 43 [0/640 (0%)]\tLoss: 0.253719\n", + "Train Epoch: 44 [0/640 (0%)]\tLoss: 0.210382\n", + "Train Epoch: 45 [0/640 (0%)]\tLoss: 0.284465\n", + "Train Epoch: 46 [0/640 (0%)]\tLoss: 0.292590\n", + "Train Epoch: 47 [0/640 (0%)]\tLoss: 0.200158\n", + "Train Epoch: 48 [0/640 (0%)]\tLoss: 0.269204\n", + "Train Epoch: 49 [0/640 (0%)]\tLoss: 0.213730\n", + "Train Epoch: 50 [0/640 (0%)]\tLoss: 0.268548\n", + "Train Epoch: 51 [0/640 (0%)]\tLoss: 0.186853\n", + "Train Epoch: 52 [0/640 (0%)]\tLoss: 0.263178\n", + "Train Epoch: 53 [0/640 (0%)]\tLoss: 0.232999\n", + "Train Epoch: 54 [0/640 (0%)]\tLoss: 0.265539\n", + "Train Epoch: 55 [0/640 (0%)]\tLoss: 0.227278\n", + "Train Epoch: 56 [0/640 (0%)]\tLoss: 0.266715\n", + "Train Epoch: 57 [0/640 (0%)]\tLoss: 0.243662\n", + "Train Epoch: 58 [0/640 (0%)]\tLoss: 0.231188\n", + "Train Epoch: 59 [0/640 (0%)]\tLoss: 0.234178\n", + "Train Epoch: 60 [0/640 (0%)]\tLoss: 0.270513\n", + "Train Epoch: 61 [0/640 (0%)]\tLoss: 0.240779\n", + "Train Epoch: 62 [0/640 (0%)]\tLoss: 0.278103\n", + "Train Epoch: 63 [0/640 (0%)]\tLoss: 0.235614\n", + "Train Epoch: 64 [0/640 (0%)]\tLoss: 0.248226\n", + "Train Epoch: 65 [0/640 (0%)]\tLoss: 0.242527\n", + "Train Epoch: 66 [0/640 (0%)]\tLoss: 0.312065\n", + "Train Epoch: 67 [0/640 (0%)]\tLoss: 0.201206\n", + "Train Epoch: 68 [0/640 (0%)]\tLoss: 0.263827\n", + "Train Epoch: 69 [0/640 (0%)]\tLoss: 0.286222\n", + "Train Epoch: 70 [0/640 (0%)]\tLoss: 0.279344\n", + "Train Epoch: 71 [0/640 (0%)]\tLoss: 0.263106\n", + "Train Epoch: 72 [0/640 (0%)]\tLoss: 0.247235\n", + "Train Epoch: 73 [0/640 (0%)]\tLoss: 0.243001\n", + "Train Epoch: 74 [0/640 (0%)]\tLoss: 0.231101\n", + "Train Epoch: 75 [0/640 (0%)]\tLoss: 0.290737\n", + "Train Epoch: 76 [0/640 (0%)]\tLoss: 0.274286\n", + "Train Epoch: 77 [0/640 (0%)]\tLoss: 0.254139\n", + "Train Epoch: 78 [0/640 (0%)]\tLoss: 0.296462\n", + "Train Epoch: 79 [0/640 (0%)]\tLoss: 0.223839\n", + "Train Epoch: 80 [0/640 (0%)]\tLoss: 0.282157\n", + "Train Epoch: 81 [0/640 (0%)]\tLoss: 0.193232\n", + "Train Epoch: 82 [0/640 (0%)]\tLoss: 0.319076\n", + "Train Epoch: 83 [0/640 (0%)]\tLoss: 0.195717\n", + "Train Epoch: 84 [0/640 (0%)]\tLoss: 0.243448\n", + "Train Epoch: 85 [0/640 (0%)]\tLoss: 0.218913\n", + "Train Epoch: 86 [0/640 (0%)]\tLoss: 0.267013\n", + "Train Epoch: 87 [0/640 (0%)]\tLoss: 0.243178\n", + "Train Epoch: 88 [0/640 (0%)]\tLoss: 0.211878\n", + "Train Epoch: 89 [0/640 (0%)]\tLoss: 0.294469\n", + "Train Epoch: 90 [0/640 (0%)]\tLoss: 0.270843\n", + "Train Epoch: 91 [0/640 (0%)]\tLoss: 0.212928\n", + "Train Epoch: 92 [0/640 (0%)]\tLoss: 0.284125\n", + "Train Epoch: 93 [0/640 (0%)]\tLoss: 0.236006\n", + "Train Epoch: 94 [0/640 (0%)]\tLoss: 0.260203\n", + "Train Epoch: 95 [0/640 (0%)]\tLoss: 0.270325\n", + "Train Epoch: 96 [0/640 (0%)]\tLoss: 0.199420\n", + "Train Epoch: 97 [0/640 (0%)]\tLoss: 0.313867\n", + "Train Epoch: 98 [0/640 (0%)]\tLoss: 0.285310\n", + "Train Epoch: 99 [0/640 (0%)]\tLoss: 0.284724\n", + "Train Epoch: 100 [0/640 (0%)]\tLoss: 0.256413\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.optim as optim\n", + "import torch.nn.functional as F\n", + "from torch.utils.data import DataLoader, Dataset\n", + "import numpy as np\n", + "\n", + "def generate_sine_with_noise(n_points, frequency, phase, amplitude, noise_sd):\n", + " # Generate an array of points from 0 to 2*pi\n", + " x = np.linspace(0, 2*np.pi, n_points)\n", + " \n", + " # Generate the sine wave\n", + " sine_wave = amplitude * np.sin(frequency * x + phase)\n", + " \n", + " # Generate Gaussian noise\n", + " noise = np.random.normal(scale=noise_sd, size=n_points)\n", + " \n", + " # Add the noise to the sine wave\n", + " sine_wave_noise = sine_wave + noise\n", + " \n", + " # Stack the sine wave and the noisy sine wave into a 2D array\n", + " output = np.column_stack((sine_wave, sine_wave_noise))\n", + " \n", + " return output\n", + " \n", + " \n", + "class SineDataset(Dataset):\n", + " def __init__(self, n_samples, n_points, frequency_range, phase_range, amplitude_range, noise_sd_range):\n", + " self.n_samples = n_samples\n", + " self.n_points = n_points\n", + " self.frequency_range = frequency_range\n", + " self.phase_range = phase_range\n", + " self.amplitude_range = amplitude_range\n", + " self.noise_sd_range = noise_sd_range\n", + "\n", + " def __len__(self):\n", + " return self.n_samples\n", + "\n", + " def __getitem__(self, idx):\n", + " # Generate random attributes\n", + " frequency = np.random.uniform(*self.frequency_range)\n", + " phase = np.random.uniform(*self.phase_range)\n", + " amplitude = np.random.uniform(*self.amplitude_range)\n", + " noise_sd = np.random.uniform(*self.noise_sd_range)\n", + "\n", + " # Generate sine wave with the random attributes\n", + " sine_wave = generate_sine_with_noise(self.n_points, frequency, phase, amplitude, noise_sd)\n", + "\n", + " # Return the sine wave and the parameters\n", + " return torch.Tensor(sine_wave[:-500, 1, None]), torch.Tensor(sine_wave[-500:, 0]), torch.Tensor([frequency, phase, amplitude, noise_sd])\n", + "\n", + "\n", + "\n", + "# Usage:\n", + "dataset = SineDataset(640, 1025, (1, 3), (0, 2*np.pi), (0.5, 1.5), (0.05, 1))\n", + "\n", + "def train(model, device, train_loader, optimizer, epoch):\n", + " model.train()\n", + " for batch_idx, (data, target, params) in enumerate(train_loader):\n", + " #data = data[...,None]\n", + " data, target = data.to(device), target.to(device)\n", + " optimizer.zero_grad()\n", + " output,last_state = model(data)\n", + " #import pdb;pdb.set_trace()\n", + "\n", + " loss = F.mse_loss(output, target)\n", + " loss.backward()\n", + " optimizer.step()\n", + " if batch_idx % 10 == 0:\n", + " print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n", + " epoch, batch_idx * len(data), len(train_loader.dataset),\n", + " 100. * batch_idx / len(train_loader), loss.item()))\n", + "\n", + "if __name__ == \"__main__\":\n", + " device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", + "\n", + " model = HyenaOperatorAutoregressive1D(\n", + " d_model=128, \n", + " l_max=1024, \n", + " order=2, \n", + " filter_order=64\n", + " ).to(device)\n", + "\n", + " optimizer = optim.Adam(model.parameters())\n", + "\n", + " # Assume 10000 samples in the dataset\n", + " #dataset = SineDataset(10000, 1025, 2, 0, 1, 0.1)\n", + " train_loader = DataLoader(dataset, batch_size=64, shuffle=True)\n", + "\n", + " for epoch in range(1, 101): # Train for 10 epochs\n", + " train(model, device, train_loader, optimizer, epoch)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc9f9031-5ee1-49f8-a70f-ad85ca015596", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 682, + "id": "90330622-8b44-4b45-8158-6840538f768c", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.metrics import r2_score\n", + "\n", + "def fit_and_evaluate_linear_regression(outputs_and_params):\n", + "\n", + " # Split the data into inputs (last_states) and targets (params)\n", + "\n", + " inputs = np.concatenate([x[0] for x in outputs_and_params])\n", + "\n", + " targets = np.concatenate([x[1] for x in outputs_and_params])\n", + "\n", + " \n", + "\n", + " r2_scores = []\n", + "\n", + " param_names = [\"frequency\", \"sin_phase\", \"amplitude\", \"noise_sd\"]\n", + "\n", + " \n", + "\n", + " # Fit the linear regression model for each parameter and calculate the R^2 score\n", + "\n", + " for i in range(targets.shape[1]):\n", + "\n", + " if param_names[i] == 'sin_phase':\n", + " # take the sine of the phase values\n", + " target_values = np.sin(targets[:, i])\n", + " \n", + " \n", + " #Not used at the moment\n", + " elif param_names[i] == 'cos_phase':\n", + " # take the cosine of the phase values\n", + " target_values = np.cos(targets[:, i])\n", + " else:\n", + " target_values = targets[:, i]\n", + "\n", + " model = LinearRegression().fit(inputs, target_values)\n", + "\n", + " pred = model.predict(inputs)\n", + "\n", + " score = r2_score(target_values, pred)\n", + "\n", + " r2_scores.append(score)\n", + "\n", + " print(f\"R^2 score for {param_names[i]}: {score:.2f}\")\n", + "\n", + " \n", + "\n", + " return r2_scores" + ] + }, + { + "cell_type": "code", + "execution_count": 683, + "id": "5eb62a22-cad8-43c4-b757-f36b6a01e9be", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_outputs(model, device, data_loader):\n", + " model.eval()\n", + " outputs_and_params = []\n", + " with torch.no_grad():\n", + " for data, target, params in data_loader:\n", + " data, target = data.to(device), target.to(device)\n", + " output, last_state = model(data)\n", + " outputs_and_params.append((last_state.cpu().numpy(), params.cpu().numpy()))\n", + " return outputs_and_params\n" + ] + }, + { + "cell_type": "code", + "execution_count": 684, + "id": "a95ee542-1c39-4f04-9184-e26c6983a018", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "R^2 score for frequency: -1.09\n", + "R^2 score for sin_phase: 0.84\n", + "R^2 score for amplitude: 0.96\n", + "R^2 score for noise_sd: 0.91\n" + ] + } + ], + "source": [ + "outputs_and_params = generate_outputs(model, device, train_loader)\n", + "\n", + "# Fit the linear regression model and print the R^2 score for each parameter\n", + "r2_scores = fit_and_evaluate_linear_regression(outputs_and_params)" + ] + }, + { + "cell_type": "markdown", + "id": "de65d0d2-b0c6-4ac5-a87f-70fa7f90480b", + "metadata": {}, + "source": [ + "# Autoregressive" + ] + }, + { + "cell_type": "code", + "execution_count": 519, + "id": "8ee139a6-aee5-4309-8685-bf0c28893279", + "metadata": {}, + "outputs": [], + "source": [ + "def predict_autoregressive(model, initial_data, n_steps):\n", + " model.eval()\n", + " predictions = []\n", + " current_input = initial_data\n", + " with torch.no_grad():\n", + " for _ in range(n_steps):\n", + " # Get the prediction for the next step and save it\n", + " next_output, last_state = model(current_input)\n", + " predictions.append(next_output)\n", + " #import pdb;pdb.set_trace()\n", + "\n", + " # Prepare the input for the next step\n", + " next_input = torch.cat((current_input[:, 500:, :], next_output[:,:,None]), dim=1)\n", + "\n", + " current_input = next_input\n", + "\n", + " return torch.cat(predictions, dim=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "272040d6-4dfb-438b-a432-744e950effd1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 594, + "id": "9f49cbf8-08e8-428a-af80-bcb2515a6327", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 525, 1])" + ] + }, + "execution_count": 594, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "initial_data = dataset[0]\n", + "initial_data[0][None,...].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 595, + "id": "c75464e3-e44d-4325-8804-29d8081e3a45", + "metadata": {}, + "outputs": [], + "source": [ + "autoregressive_out = predict_autoregressive(model,initial_data[0][None,...],3)" + ] + }, + { + "cell_type": "code", + "execution_count": 596, + "id": "90a37c56-59f3-49bc-bfbe-d9e126c42ed1", + "metadata": {}, + "outputs": [], + "source": [ + "total = np.concatenate([initial_data[0].squeeze().numpy() ,autoregressive_out.squeeze().numpy()])" + ] + }, + { + "cell_type": "code", + "execution_count": 597, + "id": "3d2ad0ee-7a0e-4b6f-8b40-dd565c8af84d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 597, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAGhCAYAAABceN/BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABehElEQVR4nO3dd3xUVfo/8M+kTXoCabRA6EgPCCGoSFNA1l5YRAWWxbKwFpQV3FXssKtf3dVVZP0JuCp2hVUUpCPSS+idhARIAgTSCKlzf39M5uZOMjW5debzfr2id2Zu7j0zE+Y+c85znmMSBEEAERERkQYCtG4AERER+S8GIkRERKQZBiJERESkGQYiREREpBkGIkRERKQZBiJERESkGQYiREREpBkGIkRERKQZBiJERESkGQYiREREpBnVApF58+bBZDLhySefVOuUREREpHOqBCI7duzAggUL0Lt3bzVOR0RERAYRpPQJSktLMWHCBHz44Yd49dVXvfpdi8WCc+fOISoqCiaTSaEWEhERkZwEQUBJSQlatWqFgADXfR6KByLTpk3D2LFjMXLkSLeBSEVFBSoqKsTbZ8+eRffu3ZVuIhERESkgJycHbdq0cbmPooHIF198gd27d2PHjh0e7T937ly89NJLDe7PyclBdHS03M0jIiIiBRQXFyM5ORlRUVFu91UsEMnJycETTzyBVatWITQ01KPfmT17NmbMmCHetj2R6OhoBiJEREQG40lahUkQBEGJky9duhR33nknAgMDxftqampgMpkQEBCAiooKu8ccKS4uRkxMDIqKihiIEBERGYQ312/FekRGjBiB/fv32903efJkdOvWDc8++6zbIISIiIh8n2KBSFRUFHr27Gl3X0REBOLi4hrcT0RERP6JlVWJiIhIM4pP35Vav369mqcjIiIinWOPCBEREWmGgQgRERFphoEIERERaYaBCBEREWmGgQgRERFphoEIERERaYaBCBEREWmGgYif2XaqAJ9vz9a6GURERABULmhG2hv3n60AgI4JkRjYvrnGrSEiIn/HHhE/lX2pTOsmEBERMRAhIiIi7TAQISIiIs0wEPFTJq0bQEREBAYiREREpCEGIkRERKQZTt/1EzmXyhAeEqh1M4iIiOwwEPEDBaUVuOEf6+zuMzFJhIiIdIBDM37gaH6J1k0gIiJyiIEIERERaYaBCBEREWmGgQgRERFphoEIERERaYaBiJ/irBkiItIDBiJERESkGQYifsDElWWIiEinGIj4KQYnRESkBwxEiIiISDMMRIiIiEgzDESIiIhIMwxE/ICjqbqcvktERHrAQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDSjaCAyf/589O7dG9HR0YiOjkZ6ejp+/vlnJU9JDuzMuqR1E4iIiBxSNBBp06YN5s2bh127dmHnzp0YPnw4br/9dhw8eFDJ01I9b/5yTOsmEBERORSk5MFvvfVWu9uvvfYa5s+fj61bt6JHjx5KnpqIiIgMQNFARKqmpgZff/01rly5gvT0dLVOS06YWFqViIh0QPFAZP/+/UhPT0d5eTkiIyPx/fffo3v37g73raioQEVFhXi7uLhY6eYRERGRhhSfNdO1a1dkZGRg27ZteOyxxzBx4kQcOnTI4b5z585FTEyM+JOcnKx084iIiEhDJkEQBDVPOHLkSHTs2BELFixo8JijHpHk5GQUFRUhOjpazWb6lJRZyxvc9874VNzWp5UGrSEiIl9XXFyMmJgYj67fquWI2FgsFrtgQ8psNsNsNqvcIv/EDBEiItIDRQOR2bNnY8yYMWjbti1KSkqwZMkSrF+/HitXrlTytCSzGouAi6UVSIoO1bopRETkYxQNRM6fP4+HHnoIubm5iImJQe/evbFy5UrcdNNNSp6WZDZx4XZsOnERn08dhPSOcVo3h4iIfIiigchHH32k5OFJJZtOXAQAfLrtNAMRIiKSFdea8WNf7czBwk2ZWjeDiIj8mOrJqqQff/lmHwBgVM8WaB0bpnFriIjIH7FHxE9J52xfqajWrB1EROTfGIgQERGRZhiI+CmV69gRERE5xECEWNyMiIg0w0DETz2/9IDXv8OAhYiI5MZAxE8VlzNBlYiItMdAhGBiVwcREWmEgQgRERFphoEIERERaYaBiI+yWATkFZV7uDfHZoiISBsMRHzU41/swaC5a7DyYJ7WTSEiInKKgYiP+nFfLgDgkU92adwSIiIi5xiIkMezZkycXkNERDJjIEJERESaYSDiQ9YdOY+cS2VaN4OIiMhjQVo3gOSx4dgFTF68AwCQNW+sxq0hIiLyDHtEfMTOrEuN/l1mfhARkVYYiBAREZFmGIiQx9hzQkREcmMgQkRERJphIEJERESaYSBCLFRGRESaYSDiIxhKEBGRETEQISIiIs0wECGHvSl7cwox8+u9OF9Srnp7iIjIf7CyKjlc9O72934DAFworVC5NURE5E/YI0IunThfKm4zp5WIiOTGQIRcEgStW0BERL6MgYgPen/9Ca/2twjA0bwSWCyMOoiISF0MRHzQP1Yc9Wr/uT8dxqh/bsTfVx5p8JjALhEiIlIQAxHCL4fyAQALNpzSuCVERORvGIgQERGRZhiIkEscmCEiIiUxEPEVnFtLREQGpGggMnfuXAwYMABRUVFITEzEHXfcgaNHvUukJG1Jc1UZ6hARkdwUDUQ2bNiAadOmYevWrVi1ahWqqqpw880348qVK0qelmQkcHCGiIgUpGiJ9xUrVtjdXrx4MRITE7Fr1y4MGTJEyVMTERGRAaiaI1JUVAQAaN68uZqn9QtyDpuwdojn8orK8c2uM6istmjdFCIiQ1Jt0TuLxYInn3wS1113HXr27Olwn4qKClRU1C2yVlxcrFbzqNYLyw7g1+MXxduMSRwTBAGPfroLKw9aa7CcvXwVT4zsrHGriIiMR7UekWnTpuHAgQP44osvnO4zd+5cxMTEiD/JyclqNc/wisurZDnOf7ecRuZF5vC4Ul1jQebFK2IQAgBvrz6GracKcCSvGFcrazRsHRGRsZgEFfrhp0+fjmXLlmHjxo1o37690/0c9YgkJyejqKgI0dHRSjfTUHKLruLfa09g0uAUdE6KQsqs5YqcJyHKjAsl1vfkjr6t8M/fpypyHqM4nl+Cm97e6HKfgSnN8dWj6Sq1iIhIf4qLixETE+PR9VvRoRlBEPDnP/8Z33//PdavX+8yCAEAs9kMs9msZJN8xrTPdmN3diG+230Wh18ZrXVz/Ia7IAQAtmddUqElRES+QdFAZNq0aViyZAmWLVuGqKgo5OXlAQBiYmIQFham5Kl93oGz1vyZq1XKDgPY1RHx86JpTOIlIpKfojki8+fPR1FREYYOHYqWLVuKP19++aWSp/UL6tX34MXXpsLJzJhTr9/S4D7OoiEi8oyigYggCA5/Jk2apORpiWRXYxGwdM/ZBvePuzYZAQEmXNuumd39T3+9FxXVTFolInKHa80YlElSOeSrHTkatsQ/LNl2GrO+2y/e7tMmBoM7xmHm6K4AgLfu62u3/w97z+G73Q0DFyIisqdaHRFSzl++3afYsZkWAVgsAp5fdlC8fUPneHwyJc1un7Zx4Xh7XB889eVe8b6C0goQEZFr7BExKK4Bo57l+3PF7f7tmmH+A/0d7ndnahuMvCZJvJ1XXK5424iIjI6BiEFV1TAQUcvyfXWByFePpCPS7Lwjce5dvdCmmXVG2I/7cjnThojIDQYi5BIvo0Dr2sAiyhyEwADXU5gToszisE1hWRXOXL6qePuIiIzMLwOR8yXl+GHvOWw4dkHrpuie9Bu9P1YRuXSlEh9tygQA3Ng1waPfaR8fgbDgQADADf9Yh3KFa70QERmZXwYih3NL8OfP9+DvPx/Ruim6d7lMnjVsjKi6xoJ+r6wSb/evN0XXlWtaRonbthL5RETUkF8GIrZv9hx2IFc2nyywuz35OtdLFEh9/IeB4vZn27JlaxMRka/xz0CkNhJhIiG5Il3R+MVbu3v1u1GhweL2BxtOytYmIiJf45eBSICfr5lCnpEmmg7qGNekY526UNrU5hAR+SS/DERsYYiFPSLkwsebswAAA1Oao1sL18tYu/OXb5QrOkdEZGR+GYhAHJrRthmG40cdSWWV1cgtshYku7lHkpu9HRvbu6W4XVpRLUu7iIh8jV8GIrZ1WhiHkDPnCuuGZWLCgl3s6dwdfVuL271axzS5TUREvsg/AxEmq5IbZwvryrPf1rdVo44x8ppEPDCoLQDg611ncLWS9USIiOrzz0Ck9v8MQ8iZhbVFzIZ1TYA5KLBRxzCZTHhyZBfx9je7z8jSNiIiX+KfgYjYJaJtO0iffjtxUay6mxQd2qRjxUeaxe2rlcwTISKqzy8DkYB6ccg/VhzBHxbvQI2FkQkBT36ZIW4/lJ7S5OM9NrQjAGBPdmGTj0VE5Gv8MhCxdYhkXryCXacv4f31J7H2yHn8duKitg1z4mIpS4SrSVqSvXNSZJOP1zc5FgDw84E8FPlxyXwiIkf8MhCRzkOdvGiHuF1ZbdGiMS59svU0rn11Nd5adUzrpvil4MCm/xOJlcy6Wb4/t8nHIyLyJX4ZiEgLq1bV6Hs45vmlBwAA76w5rnFLgBqLgK2nClBR7buzP86X1M2WGT+wrSzHDA8JEre/ZcIqEZEd/wxEJNsCM1Y9tizjHH7/n62Y/d1+rZuimDdWHBW3Hx/RSZZjSiv47jp9WZZjEhH5Cv8MRCRdInovJaLHZXG+231W6yYopvBqXQ5HpDnIxZ6eu6Zl08rDExH5Mr8MRAIkF3edxyGkspDanJCgAJPdCrpNOmZQAL59bLB4W5oMS0Tk7/wyEDHBcSSix94HHTZJl574Yg/uW7AFliZOwbbNUHprXF8ZWlVHOvvmvXUnZD22L8m6eAV7sjl8ReRP5Ol7NhiTXRyi7z4Rk8mk//EjHViWcQ4AcCi3GD2bsK7LtsxLAID4iBBZ2mUTHlxXnZUL4DmWXVCGoW+ut7vvvfv72S0eSES+xy97RIyEPSLeaUrMJk0kTYgyu9jTe0GSacCtY8NkPbYvOJpXgiFvrGtw/7Qlu/HvtdrPGCMi5fhlIGLXIyK5cFXV6K+OiB6Hi5xZsi0bm0/qsyicJ+6ev1nc7pTY9EJm9T0+ojMAIL+43M2e/ufhT3Y6fezNX45h4GuruUglkY/yz0BE0s9QLckpeObrfeL27O/2YebXe1VtlyMmg/SJZOQU4rnv9+P+D7dp3ZQmCwsOtJtZJRdbcLP/bJHsxza60wVl4vafhzecNn2+pALf7GINFiJf5J+BiJNrjG3svuhqFT7fnoOvd53Rvry6MeIQ3XzLb0rOT5821tySN+7tLVdz7AzuGIcAE3DwXDFyi64qcg6jigix5tB8/6fBePrmrjjyymhkzr0F9/ZvI+4z85t9XA+KyAf5ZSAS4OLb7onzpXYfdlr3BhskDkGYJBnTqBeLgiuVAIBWCuVwxEeaYXtp5q8/qcg5jKq8dnkF22sfWtsr9ca9ffDK7T3E/bacLNCkfUSkHL8MRFz1un+7+4zdWLTWORpan99TYSF1gcgVgy53X1BqDUTiZJ4x48h/t5xW/BxGUVVjEYPX0KDABo8/KFkB+YGPtuHkhVK1mkZEKvDPQMTN49Lv8656T9RglByRQEmVuLIK461Fc+pCKa5WWdsdFynvjBlneEG1sr3uAGAOdvyRtHjyAHH7X6s5i4bIl/hnIOLm2i5dGyRARz0ix/NLtGuIG9JCYnpcxdidpXvqytbLVdrdkedu6SZuj3p7o2LnMZLy2kDEZALMQY4/km7onCBu/2/vOVXaRUTq8MtAxF2fiDQvROseCenZb3p7I37W6TLy0rQQvReJc+RibX7I7xQuniVd0bfaoLk0cisqs67vYw4KcDpbKTDAhHl39RJv/3dLlhpNIyIV+GUg4qpHRBDse0S0vqjW/2B+7LPdGrXE3oF6U1D1kuDb2HMX1M6OSmvfXMbWNCTX+jW+orrGgptqe4bcDYPe3re1uP3x5iwlm0VEKvLLQMTdB570YsZZM45tPWU/e8Gi9QvVRAfOFgMA2sVFaNwS/3Ikr264sazSdW5RWEgg/n1/KgDrwoHVOixASETe88tAxJtkVc0vr3qNROqx6xHRsB2NyS0+cLYIZwutdT2ask6Np94Zb72Ydoj336DHYhEw6PU1+N27m7z6vWFdEwEAxeXV+M+vp5RoGhGpTNFAZOPGjbj11lvRqlUrmEwmLF26VMnTecxtsqrdMIPGQzOant25+kNGdsNZGr5mjTn1M5IKus3ClR866ZoUBQA4dfGK336rX74/F3n1iuDd0Dne7e9FmIPE4mcfsBYLkU9QNBC5cuUK+vTpg/fee0/J03jNXQKqnkYZlCg1Lof6rdJyaKapgY90lo8ar3d8ZF2dEulCe/7k4Lliu9ujeiThgwf6e/S7nz88CIC1V2TDsQuyt42I1KXcPEUAY8aMwZgxY5Q8RaN4M31X65hEp3FIg2nN0i/2Wr9m3qqsbfy3jw1W5XzSOiVnLl9Fmipn1Rfp38/eF25GjBc9Ub3bxIrbW04W4MYuCc53JiLd01WOSEVFBYqLi+1+tFAj6GMGCGDUoRm1W9M0tumjsSoMy9j8fkAyAGsg4m8EQcCJ89ZibiGBAV4FITZPjrSuZLziQK7dUCoRGY+uApG5c+ciJiZG/ElOTlbkPAFuqpQJOp6+qxf1m2V/MVD3NWtK4FNjEVBSu9hhbJh6gUibZtY1VbIKrqh2Tr149tt9+OVQPgDg3dpZMN6acn17BAaYkFVQJiYaE5Ex6SoQmT17NoqKisSfnJwcRc7j7tIu5zV147ELSHt9daPHsvUZhtS1q7LaggslFXa9SEZScMVaPyTABESrGIj0a9sMALD6cL5YWdQf5Fwqw1c7z4i3B6Q0rm5LVGgwWsWGAgBWHsyTpW1EpA1dBSJmsxnR0dF2P0pw18kg51TUhxZuR35xBSYu3N6o39dph4jYU3PLO79iwGurxa52QOOCZl7un3WxDACQFB2K4ED1/jkM6hCHuIgQlJRXN0jc9GU/H7CvDNyUWUpRZuvvvrr8sF8Fc0S+RleBiFpczZoRIOisOJc+IxFbgGQLQJbvq7vAqP3qNeV893+4FQCQW1TuZk95BQSY0KV2Gu+Zy2WqnltLr/90RNyecVOXJg093ndtG3F788mLTWoXEWlH0UCktLQUGRkZyMjIAABkZmYiIyMD2dnZSp7WLfd1ROq2tY5JLtaWHteb+sFcjUETBrVc7yU6zDpprbi8WrM2qKmi2r7X4vERnZt0vIfSU8TZN5tPFLjemYh0S9FAZOfOnUhNTUVqqjUhbcaMGUhNTcULL7yg5GndchWHbD1ZgDd+OSre1jpZVa/qB3PVOllrxhvS+iEPD+mg+vnDQ6yByPbMS6qfWwvvrT0hbn808domHy8gwIS37usLANjhp/VYiHyBooHI0KFDIQhCg5/FixcreVr3XEQie88UYaMksdQoF1VvDO4Y1+Rj1H8Jpd926wdvFouAt1cdw53v/4bd2fJfMBpb0EwaACi96q4jJbU9IT/sPad5BV81SEuyD++WKMsxr2lpzSPbm1PIPBEig/LLHBF3i95J+eLl4Z+/79vkY9R/CfOL64aQ6l9Tv99zFv9acxx7sgtx1/ubm3xuuRSXV4nbnROjVD//xMHtxG3bFGJfVVltQXmV/BVsW0SHituPf75HlmMSkbr8MhDRZ/qneiLNTS+o665MvtSx8yXud9LA0dqVX0f1SEJY7folarqhc4K4bsql0krVz6+my2V1z+/nJ26Q7bi2PBvAP2uyEPkC/wxEvOkR8cEuc296hJxycQi1X7LGnu5fa44DACqqtVt4LrH2G/05Hy/K9fKPh8Rt23CKHEwmE5ZOuw4AcCy/FOdL1J39RERN55+BiBf7ahmH6Ll0tatgxmgJvr1ax2h27s6JkQCAo/n67DWSi3R6t9yuaVk3rLbyYL5i5yEiZfhnIGKAsZn84nJ0eO4nh4/lXGpa3QmFO0Q05WkPliAICKktYDZugDJLCXiiVay11PuFEn1O05aD0r2K5qBAPHpjRwDAsTzfDuiIfJF/BiJeXEa16hFZ+Fum08de/N/BJh1bjqEZV4eo/5p583o3RmPeo6tVNeKqu83CQ2RukefiIqznvnTFd3NEpDOlnv9dd0XOYetZOq7TfCQics4/AxEvnrVWwwxhwc6TJ5s6w0KOsECWPJMmaOqw1cWSugt/uAaJqjYtYqw5Il/syPHZYOS57w6I21Oub6/IOTonWQORracu2U2/JyL9889ARIFjnrpQitnf7cNpmTL3Xc5saWJsJMfUSVeHUHoRsqe/2ovr/r4WJZLpt956/Iu6qZ5arnB8rWTRtz8s3qFZO5TUISFC8XN0TIgUt19dfsjFnv7hp/25itTsIVKCfwYiXs2a8Wy/cf/Zis+352DSInkuJkouwKb0ZfddSQVNJXy7+wxyi8rFBMjG9Fpl5BTK3KrGaR8fgahQa9ApXTjQV1y6UomfD1gD03/JUL/GmQhzENrHWwOets2VD3z0av3R8+jxwgr86bPduOv9zUiZtRxbTxWguLwKS/ecRamP16shY/LPQMSLfT29xNmSDTMvytMjEhepXN6CHB0AT3yRgZ1ZzkuTf7/njNPH5BIQ0PgnEhxo/d3b+7aSqzmN9tHEAQCAqhrtphEr5bOtp8Vtaa+FEp4caV27ZvXhfL+84K47ch6TFu3AlUr7CrO//89W9H7xFzz5ZQYe+WSnRq0jcs4/AxEvrl9a1BH5bvcZTF+iXJVIuYYi7vlgi9PHnvpyryzncCXQwfPw5N06X1yOqhoBASbg1Tt6yt8wL6XEhwOwBiJGXTzQmfOS2UCdEpUNRHpKpmFvz/SvRfC+2pGDyR4M7f12ogCFZZU+WR+JjMs/AxFvZs0o2A5nZnzl+iJutDodSgms7RHx9jM1t8ha9KpFdCiiQoPlbpbX4iLMCDABFsG3Zs/syLqET2p7RG7r0wqhLhKw5SDtcTlX6D+FzTYcu4C/fLvP7r5HhnRwOkOp78ur0H72Tzhf7D+vEembfwYiXnQITFy4HTtcDEGQuqSzZRo7NHOltts+MrTppe7lEBhgQvMIMwD4VGXQd2or1wLAiGvkWeTOncnXpQAA/rb0gOsdfcQzX+/FxIXbxduhwQE4+fotmH3LNZhyfXv8/MQNmD6sEzbPGt7gdz/dlq1mU4mcYiDixpnLV3GviyEILShdl0PPqix1eRSNTRGxTX+OkGHNHbkkRFkDEV8qbJYkWZAuvUPTV3z2RNvm4eK2dEVoX7Qn+zK+2WWfi7XiiSFiTyFgLaf/zKiuaBUbhmvbNbPb9501xxWf4UbkCf8MRAx+IffnoZlKybowQY2MRJ6onborx+J/cvHFQGT/mSIAwP1pbcU1dZQ2Ia1uRWO5Esf1SpoTMmtMN2TNG4uUeOczhr55bDDWPTMUd/VrLd73yCe7cLHUd/7myJj8MxAxdhzi16pq6oKwxiTdni28Ki5HH6thRdX6EiJrAxEfuSiUVlSL6+f0VnEtn5CgAPRJjgUAZF7w3UBk3s9HUFhWV0fnofR2Lvau0z4+Aq/d0cvuvmtfXY1dpzn87I2qGgvWHTmPl344iM0nL/p875vS9POVUEWMQ9QlZ+AnneLamMz/Qsly9O3jwl3sqS5bj8iJfN+oJZIl6Y1QO+DrGB+BvTmFOHnBN17L+i5fqcQHG06Kt4++OhrmIM8TgcNCAq29J7OWi/fdPX8LsuaNlbWdvubUhVJ8vj0bH/5qv/zGot+yxO2R1yTh73f3QlztFwvyjJ/2iBg7FNmR5b8VEy2S4KO6NnHVm3jkYmldIBKkYNE4byVFWz+4Vh3yjdVjsyULM97UPUnVc/dqY+2B2eCDpd6raixIfWWVePum7kleBSHkvUtXKnH/h1sx/P82NAhC6lt9OB/9X12N1386zFlJXtDPJ7GKjB2G+Ddp0NGYmhsXJTkYQ7smyNEkWQzrap1VUlJR7RNj9lm1Sx3cmdraLnlSDaN7tgBgDdjzinzrYiAtEAcA70/o1+hjrZ4xBKN7tBBvT/3vTtYXqWfX6Uvo98oqbD5pX5cmNDgAn08dhHfGp4oLLkr9Z+MpDHx9DfIZjHjEPwMRhT8XN5+86FP1IPRE+jHpKBBx9zlqu8h3SoxE7zax8jWsiaRJhh+sP+liT2PILrD2iEhnsailZUwY+tfOELnz/d9UP7+Sdp6u6w3dPGt4k5aC6JQYhQ8e7C/eXnUoH4dzuXoxYB32PZxbjLvn28+YXD3jRmTNG4sjr4xBesc43NanFVbV3nfo5VENvtykvb4Gh84Vq9l0Q/LTQETZSOT+D7fh5rc3KHoOkgzNeDGLyDYrZZiOekPqW3XY2MMzZZXV4hBTO43ycK5NsQYiuT7UI2KxCPixdn0lwFqQTw4v395D3N5/tlCWYxrdsoxzGPOvX8Xbc+/qhaOvjnZZHTg8JAiLJw/EsVfHIFpSo+iWd371mSFXpfhlIKIGaS6CXgzpkoD5TejK1QNp13GjhmZqe0TidZhMZlsrpYOLKZhG0P2FlSio7RHUKhB5KD1F3PaV4YY9koUaFzzYv0lrLUk9lJ6C8QOTAfhmXo238ovL8eSXGeLtZdOuw/iBbT3OxQkJCsC+F0fhgwfqPmun/ncngxEXGIj4kRdv7Y4xvVpq3QzZ1DQiWfWCjgOR3rVJlr4yhRdQfqE7Z+Ii6mbq5Fy6qkkb5HS+pBx3z98MwPq3e7PMCcC39rEu/vjT/jz8tD/Xzd6+60pFNdJeXyPe/nRKmjgd3Fuje7bEwknXiren/ncnftx3rqlN9EkMRPyI0WcLAU1PVj1z2XpRatMsTK4mySYh0trV7ktFzbSq1RIaHIhuLaIAAF/uNH4p8x/31gUHd/dvLfu/5f6Sqqt/+mw3tpz0r0UDAWvP2R3v1eUUTUhri+s7xzfpmMO7JeH/PVQXjExfsocJrA4wECHDmvO/g5jvRWJnjUXA2dpAJFmDJEp3bLVELpZW2q2pYyR6Wj343mutww3HfaA2y9KMs+L2/QPbyn58c1AgFk0eIN4e/+FW2c+hZ4IgYPqSPTh+3vq38sSIznjtzl5ufsszI7sn4Zenhoi3015fg7LKalmO7SsYiPgRtftD3lt3QvFz/H3FEY9TVXOLrqLaIiAkMMBuHRS9iIu09h7UWARcLtNfjpEnXlimn8XmOiRYc21+OZRv2MAOANYdPY99teXyf/zz9WgXp0wO0bCuiXZJ3P50sfxxXy6W1w5JDe+WiKdu6iLr8bskReG5W7qJt7u/sNInpunLhYGIHwkPUbfw0Rsrj8r+YdaUvENbka3WzcJUr23hieDAAHH9m/VHjZk0+JlkRdcUjSvXtpP0etVfHM5IPtlSVzukR6toRc/197t7i9v3frDFbm0nX7Xr9GX8+XPr+lMjuiXio4nXuvmNxnl4SEe8LullufbV1ciRFP7zZwxEPPTA/9uG3CJjJ72ptfCYVFW1IGu2eFMW/LONe+txWMamtHZlYFtBMCP7fwp9oHtKWsPkpR8OatiSxiuvqsGm4xcBAMsfv17xPC9pafKD54rx7W7jBnCe+vOS3eL2vLt7K/oa35/WFuNqhwwB4Omv9/rMrK6mYCDioU0nLuJv3+un29koFm3OxInzyo7Re/oP+d211qGic4X6DSifudnaJXxWx2105rQkeOrTJgadEqM0bI19Cf/+Kc01bEnj7TtThMoaCxKizOjeUtneEAAIDDDhl6eGwBxkfe22nvLtpNVTF0pxrrbWzLy7eol5Wkr6+z29cU//NgCA7ZmX8PVO3w/23GEg4oWLjaiW6k1wXVpRjbdXHfP6HHIIC1Zm2OaXg/LOnW/sl4d9ZwrF7es7NS0TXkm23ho9B0vO2BaZCwww4etHB2vcGitbLYfiq1Vu9tSnV348BAAYkNJMtVlvXZKi8NFEa+LqsoxzuOyjVaIFQRBf30hzkJjcrIZ5d/USi6P95dt9mn3u6wUDEYV589Hx+k+H8a81xxVriytP3yxvcpa3BEGQaVqb40jlng/qSjU/d8s1MpxHGa1irdOKt566hJ1Zxlqa3TY1ekS3RIQE6eOjxRbYnbxQqqsZPZ7YmXUJ+89ak1QHqNyj07dtrLj9/Z6zznc0sNv+/RvW1eZifTJloKp5Y0GBAfjy4UFiBdZ/rTnuc+sieUMfnxYEANgtWUdCaatnDHG/kww8/eh/a9UxpL2+Bp/UW9TLk+N5cg5p0p1eLpKO2AIRwD54MgJb4l2bZvrJwemaFIVIcxBKyqsNt+bH7uy6zwO110WKNAeJhbze/OWoqudWw+aTF8Ug74bO8Uht28zNb8gvLtKMjybVTZm+7u9rcaXCf2YqSen3E9lHeNOdalExaUmt8Xtv8zde+p/rpEJfT+xKUmGMWim2CqbJzfVTLC4oMABp7a29CRuPG2sm0tZTdT1i/SQ9FGqZeXNXAEBZZQ2WbDN+UTip+z/cJm5rmVQ9IKU57rvWmi9SYxHQY85KzdqiJQYiCvOms89gPcdNVl5V0+C+MJWnGOtNUL3VVI0UeOVctvaIJOuoRwQA0jvGAQDeXXvcMK9neVUN1h45DwD45tF0TaoiS4dnnvt+P0rKjZlnU9/RvLoVhl/4XXeP15BRyj/u6YPWkp7QT7ZkadcYjTAQ8YYg4Ps9Z/DFdmW+HRi56JIzznp59uYUotvzK/BqbbKYTURIkMP9bRwOzXjwsg3vlgjAuvCfkZRVNgzW9Mo2NKO36dGjerQAAJRXWcTF+PTurdrkxfCQQNWHZWwizUH48uFB4u1/q1CgUGk1FgFT/7tTvD1xcIp2jZH46YkbxO3nlx30uzVpGIh4obJGwFNf7sWs7/ajwMOqeM6+yJRWVDc4hppDM2px9pTeWGkdd/5/mzLt7ndXdK0xL9GJ86Xit8uxvVp4fwCV2dZIAYBCg8z2OF1wBcXl1vFtva3jk9w8XPzGufbweY1b45n1R63t/OMNHTTNaUrrECduL9hwSrN2yGXbqQKxsOGyadfpprBhTFgwvn2sbqbZ9CV7UFFtnC8hTaXKX/h7772HlJQUhIaGIi0tDdu3b1fjtLK7UFKX1bzvbJFHWfgmJ4MzPeesRP9XV6NY0t1Zo2EgolTXr7Nn5KwwWaiDacSrDuXjnTXOu9Uf/KhuvNfRLq8tr+t10XOiqs1Pj9d9OzLK1EnbNMg2zcIQYXbdq6UFW6/IHDc5SHpw6kKpWHtn/ED1ppQ681fJLLNLBvl7dKToahWe+34/AGthscauqquU/u2a2fVA/WHxDofD175I8U/lL7/8EjNmzMCcOXOwe/du9OnTB6NGjcL588b4ZiJ1sbTuH+HkRTsw85u9TT6mdEEuiw9WU/Z2TD5A8heZV1SOorIqTP3vTry16hg2Hr8IR6GNbR0OZy6X1QV7I6+Rd/l0JQQEmNAlyVpjoMggPSKra3sabFN49eaG2lVUr1bV6H6a5Nurj8MiAOkd4tAyRvvepT/e0B4htblL/V5ZpXFrGm/Sou3IKrD2hjw6pKPGrXEsrUMcXrm9BwDgtxMF6Pb8CsPkNTWF4oHIW2+9halTp2Ly5Mno3r07PvjgA4SHh2PhwoVKn1px3+32YH69g46GN1c6ng6n5R+cUh2Uzp6Rs54i2/2Xr1Ri0Nw16PPyL+JjntQZcdSxc12nuu7lqNBgt8fQgwsl1mG7j+oNXemR9O/25doPUb0ZLPkbOHVRv6vxZl68gh/2WvMD/nB9e41bY2UymexePyNeGIuuVolfWO7q1xptNV4HyZUH01MwfVgn8fbERTs0bI06FA1EKisrsWvXLowcObLuhAEBGDlyJLZsaVgjoaKiAsXFxXY/Rlf/uphdUOY06csHc1UbXQn1WH5Jg/tMHhzP0eO2+yZfl9K4xmjA1otjy23Rs/O1QVOACfj9APmXqJeDOSgQQ2tXls0u0O9CY8sy6r7cSANorb19X19xe+Y3+7RrSCN9vTNHHEr/h2RhP716ZlRXNAu3fmnaeOwCtmcaq7ihtxQNRC5evIiamhokJdl3hyclJSEvL6/B/nPnzkVMTIz4k5ys/fio3K64WI3WF5NVMy96t3ibrUfDWRJZY16hktokSq2n6XkjVYO6EY1lW4+kc2KUrnNwbIvgHcvXb4+IbZZUtxZRCHczg0xNzSJC0DEhAoB1JWMjzfA7V3gVry4/DAB47c6eDabI69WmZ4eL2/ct2IL3fGDWkjO6ekdmz56NoqIi8ScnJ0frJjVZ/aECV7GGloGIBmUKHLI1I0DGbHZbtVYjrZ787vhUANZehqIyfeeJ2OoyDGyv74Xl0mtngKw5Iu/6R3LKqg3cxw/UX8/S/0l6RYb933rN2uEtaS7feJ322DkSYQ7C7udvEm+/sfIoDpx1nQ9nVIoGIvHx8QgMDER+vv0//Pz8fLRo0XAapdlsRnR0tN2P0TnLhXBEz18ybu/bSp2u4tqIKMBBZGQymRo91ANYa5cYRZtm4WgWHgyLAOTJsgaPcg7lWodQO9R+Y9arwR2tCaunC8p0WUr7amUNfjlk/axsp8Mchj5tYpBYW/n3dEGZIXpFzhVexW8nrD12d6a2lvULjhqaR4TgFknJgd+9uwkbjhmrQrAnFA1EQkJC0L9/f6xZs0a8z2KxYM2aNUhPT1fy1LpVf9qqtIqenodmpg3rhM/+OMj9jjIJdBSIwPm0X2eqa+qmIv3dAGPDUrHhIQD0PXOmsKwS62sXDlN7YTZvxYQHI6p2avFbOlzt9LNtdesstYvTX1BnMpmw4sm6Naqk7dWrwfPWituv3dlTw5Y03vsT+mNs75bi7YkLt4vFA32F4kMzM2bMwIcffoiPP/4Yhw8fxmOPPYYrV65g8uTJSp9aFdIaFY64G/JYmnFO/Gah5eqg7r4nBKn8TUKuoaLc2qmawYEmXKvzC2V9tpU5t2cWaNwS56QLovVsHaNhSzxTUtsT8tGmTF3N/iivqsFCyQypFB32iADWb+h/Hm6d0fH8soM4V6jf4U5p2966r4+ucm689a9xfe1u3/CPdfhpf642jVGA4oHIuHHj8Oabb+KFF15A3759kZGRgRUrVjRIYDWqD391Pb2y/vXU4awOF495yzYz5Pa+rZp0nD/WmzoYrFKCl+31cpSsajJ5MGum3m3blL2uLaJ0U0XRU3tr2/7mL/r79m6zxiCVSm3mT+gnbucXe1YdWQ1fbM/Gudqg+de/DNNkbRlPTZKURf9rbYEwPXp/fV1y552prTVsSdMFBQYga95YvHd/3d/vnz7bjU3HL2rYKvmocnWZPn06Tp8+jYqKCmzbtg1paWlqnNYwbN/MGtsj0q1FFIZ0ScCCB/vjhd91x/pnhuL1O3t5dYz6H3zt6433q30Rd5Qj0hhrDlvH3Hu01P+39fr0mLBYny0BdIpOal64M6ZXS0TULiNwOE8f5QEEQcCLP1h7Vkd0S9TdWj31xUWacU9/64qx645ewKkL+puFVF1jEcv5Pziona4DO2+M7d0Sc27tLt5+4KNtSJm1HIVlja94q4eeQV3NmvFFnvwDsMUfjc0RSYgy479/GIhRPVrAZDIhJT5Ctgu5TVCgOv+Q66bvOn7c25cos8A6C8FI02FtHh7SQdzWctjOFdsSBZ0SIzVuieeG11bXPZLbsFaNFq5IFjZ0tMSBHv1ldFdxe8RbGzRsiWMLNp7CuaJyRIcG4a9jr3H/CwYy+br2uD/N/ktK35dXYe7Ph71OID5beBW/e3cTVhzI1TQgYSCiMM+GZgSnj+lFcIC6QzPOAilvk1VtsyPa6vxbpiO2GQoAsO9MoXYNcUIQBLG0e2yYMSrWAnWLCh7RSY/IhqN1syCeM8hFMzEqFEnR1r9PQdDXGjQV1TXiopp/GtbJMMGdN16+rUeD5NsFG06hw3M/YdqS3ShzVa/KIuDfa48jZdZyXDdvLQ6eK8Y/Vx/X9MuOcbN3dGTU2xub9PtCE3tE5OCuA0WtHhEbRz1J3nbynLlcJhav0uNCbO5EmIPQLDwYl8uqdDlzZr3kAlp/KE/PrmlpDUSWZZzDW/f11Tx3aNqS3eK2bZVgI3h/Qj/cPd9aIfubXTl4WCfrtzz2qfX1DA0OMMyQobeCAgMwIa0dJqS1w/d7zuCpL+tqpSzfl4vl+3IRGGBCq9hQRJmDxSn2zrx5bx9NC72xR0QGRx2UIxfVL2jm4Bt9RbWl9jH9Ui1Z1UW0YYJ3dUS+3nlG3DbqtyLbkMfVSv2twrlX0kvTJTFKu4Z46ZqWdfWJtC6dLZ3ZMTG9nYYt8V7/ds3xUG2bX//piC6GD89cLhOXRXhqZBfVPre0dGdqG/w2a7jY02dTYxGQc+mqyyAkMcqMY6+O0XzGm++/Sxrz5LvW019lANBH0pBN/aaoPX1XDtKnEBtunKEDqXOF1pkUtuqwemKbdfLEiM6GKhQlXdF21SFtq6xKF3K8oXOChi1pnLv7tRG3bRV2tfTOmuMArDlhj9yojx4aNbSODcOKJ4fgt1nDPapw/MY9vbHrbyOx/a8jdbEsg/H6qw3OUayxWoEpkN7mUri7jKjVfW07i6OgzJOhGemv2fJDOiVGIik6VIbWqe9s7TfmzSf1V0vElrdixNd2/MBkfL49Byc1nvHxiyQQGtYtUcOWNE6f5Fhx+9TFUnRvpV017JxLZfiqthf0wUHG6l2SS+vYMHz1iLVYqMUiICDAhLLKaphgQlhIIC6WViAuIkR3s4i0D4V8nDdveGP/OBr7e9//abCLY8pzDrl502lUXJtXcVc/Y9cQ0KPDucU4eM7a5WtLWjSSW3pZK1UeySvWrCeyqsaC+etPAgBmjuqqea5KY43uYS1BrnXC6qLfssTtm3s0XELE39h6KcNDghBWO2U9PtKsm89yKQYiCmuw6J2LfRv7gdjYPNLUts3qbujkj7OpzZC+hrappdGhxhyWAYAPHujnficNfLOrLv/GiDOSrmkZDZPJOrx0RKMhBVsgBwDjBhh3pfG4SOtSBPkarol0paIaC3+zFpf8v3v7INKAyen+jIGIypT49mXUb1KO2BYJdPYqeTPkZJsxE22gqaX1DelSlzdQUq6fmTPS2R1GqiFiEx9pFoux7chSP2HVYhFwx3u/AQBu6p6E+Ejj9SrZdEmyJkl+ti1bs4TVHnNWitu3NbGqNKmPgYjC6ocISvw7lbt4GaDfmiaelng/kleMzNol1aNCjfvtKDwkSEwmO3nhisatqWOb6XV3vza67Or1RFp7ayCyfJ/6a3ZIZ9r1bGW8qr9S4wYkIyYsGIVlVZoshHfwXJG4PeX69n4xU8bX+O07NrijCkvao2FuhV57RNS+lDjt2ahtiKOXyZsL3u7TheJ2cjPj1GZwpLL2oj+jdnaVHtjqmkSHGTfIu7GrtbdpW+Yl1Xubsi7WBZX1q2QaTWhwICbUPod5Px8Rk8TVUF5Vg7HvbAIAhAQGYOaorm5+g/TIbwORT6ekYf+LN6t+Xmc9Il/vzGl0b4mjqZOhQYF2+RY9vMxmV/JLbmODMRPc11qxHTq3yDrbpHVsGDoZqMaFK6d00iNSWlGNDzZYkyxbxhhvxoyNdIVbWxEstRyo/RZ/Z2prJEQZd1jG5s/DO6NlTCjKKmvEBFw1LNmWLW5/P22wYesF+Tu/DUQCAkyIUiGJseHQjONL6cxv9jX6HIEOooaAABMOvTQau5+/Ce+MT8WnU7xbaFDJoZkdWZedPlb3TJrWgIycQgDAozd2cL2jATw+ojMAYGhXfdSZ2JNd9/6lxBmnomp9seEhYsBdUa1ewbgrFdV4b531Yt3PgGsgORIWEij+fX68OUuVmUhVNRa8/KN1scCEKDN6GHyIy5/5bSCilkv1VkX0dlEiTzgbmgkLCUTziBDc1qcVmkWEuDyGmsP8lxVcKdI25GM7R2uDD8sAdb1ZeinzLm2HEWtfSH08eSAA18Gx3H7cd07cvrWP7yRW/nm4NWAuqajGP1cfV/x8b/5yVNz+4uFBip+PlMNARGHS6+ai3zINk6yqpC935Dh9zNVTMZk87ye5UmH9hhsRYtwcBhvbgnJFZfoIRKYv2SNuGz0xMFFSAyXzojpDX//dYk3oHNIlAbHhrr8gGEkryUyqf6057nLhtaY6nl+CBRtOAQDMQQHomGC8mVtUx9ifIgbz0g+HFJlrL8e1wKRiuuraI+dRUu74Q0qcvtvYgK3290prE+aMuNhdfbaL1amLV3QTjPiKrkl1+UO/HMxT/HwXSipw8FwxTCZrvQtfs3X2CHF7yuKdipyjqsaCmyQLja6ecaMi5yH1MBBR2dNf73W/kwuO1nzRqo5IU85bUWVp1O95EqDszLqECyXWdVB8obBRjKQOyqLNmRq2xHoRsGlhwNLu9ZlMJjxRm4Mz9+cjds9PCT/stQ7LdGsR7RNJqvW1iAkVc5q2nCqwmx0kl8mLdojbiyYNQLIBC+qRPQYiBvPO+NQG9yk9NNPGSZ7FMzc3nCp3T/82DvZsyNn0XdtTcfSoyaN5M8A9H2wRt1sYeFaHTaLkgnXgbJGLPZU35l+/ituv3NFTw5bIZ+LgFHH7481Zip1HEAQxuXJUjyTFzqO1J2sDEQCY/vluWRNXf9qfi00nLoq3r+8cL9uxSTsMRAzGUcghR4+Iq/VCVjw5xOPjvKlxd3P9jzxfmM4XEGDC/AnWUu9KLJDoKUEQcOJ83SJx4SHGf20BoHlECGz/hPYrGOhJj22r6uqLAgJMeOm2HgCAA2eLsf7YBVmOW1VjwZ8+q5tmvWHmUMPnKJEV30UFKTGFzVHnR1N6RBY82B9/Ht4Jw13MfnA2vNGUjhhnOSlij4jDgmbA//1yrPEnNTBpros0GFCTrZqqjdErgkotePBaAMAhyfovcssqKAMAhAUHIs2HAxHAuvptVO3f7LTPdovrPjXF3J+OiNuP3NgB7Qw8dZzsMRBRkFpl0psSiIzq0QJP39zVEGW6TQA2nyxwuY8gABG139T1umBcY0RKytSXV6lX80JKmtczolsiYsKNu4ZPfX2TYxEYYMLx86XYd6ZQ9uNXVlvwl2+s+WGje/r+yrABASasnzkUbZuHo6yyBlMW73D/Sy6cLykXF7UDgNljrmlqE0lHGIgoSJk4xFGyqiIncqspoYvTHJEmzt7JKy7HlUrrhXpwJ98ZP05NjhW3H/tslyZtkBb9SuvQXJM2KCUhyoxera09PC//cEj24689ko/y2kDO2yrHRhUXaRZLv+/IutzoNX3Kq2rwx4/rZuBsenaYLO0j/WAgoqDTBfJnjDscmlFx1owSp7KVYwekyaqNC+PeXGktchQTFoxoFSrnqkXaY5Vz6aqLPZUjHZqZNLi9Jm1Q0pDaxMc9OYWyV1pdI8ntuaVXS1mPrWeP3NgRd6a2BgBMW7Jb7BXyVM6lMvzu3U3Yd6YIzcKDsf6ZoWjTjLNkfA0DEQU99WWG7MesHweYgwIwSZL1LxdnYYASM3TS566V7Vh5tXVafGHart7YhoSahQeLKwL7kidGdgEA1FgEWWcnFZVV4etdZwBY1z6SFv7yB3Nu7Y6era29QF/tPIO3JBVRXdlysgA3/GMdTpwvRURIIP7z0LVIiWdeiC/yvU8THTlfW8tCTtJvxtOGdcSBl0ahZYx6H2zSQKQpMYm7/BlHjz+/7KDHx2cgIj/btElfmInkiHT22RsrPbtYeuKz7afF7ed/53+5DbHhIVgyta4E+ztrTyBl1nKcLXTes/fljmyM/3Br3e+MT8WAFN8aDqQ6DEQUpESyav1rv1LT15zFGNLgQ81qrABwsdTzwC4u0ndKZ9tIa8goUSjKnYO1M0o6JfpuOe0HB7UDAGw9dQmFTVgTyaa6xiIGNTNHdcXonv4zLCMVHRqMAy+NsrtvyuIdWL4v1y75+vPt2UiZtRzPfrtfvO/Ne/tgxDW+W3eFAH5tNBi1Jrc4i6HUquLa1CAuLtL3qlbe0rMFHq/d3nj8gqrd1IIg4Jva4YWHhxh/RWNnnrvlGnyy1dqDsei3LDx1U5cmHe/+D7dBEKw1V+67NlmOJhpWpDkIu5+/CS//cBBLM87hSF4Jpi3Z7fJ31j8zlMMxfoA9IgpqbMKlnsk2NOPkfrmmEbfzwbLPQYEBeGCQdRbCmcvqJqxOkcxa6COZweNrwkIC0a2Fdf2ZFQfymlQLqKC0AtuzLgEA0to398mS7t5qHhGCf/4+Fd//abDL/YZ1TcCp129hEOInGIgoSJGhGY3Lfch1fouTF0eup/fo0I4yHUlf2tYGWOcVWDzRlbVH6mZ9+NJsJEe+fDgdwYEmHM0vwfFGFo/LLy5H/1dXi7f/89C1cjXPJ6S2bYZjr47BlOvb44FBbRFSO8Q8c1RXHH9tDBZNHqjqbEDSlt8Pzbw/oR8+2XIaW065LpTVGBZFckS0/ccp26wZJ69NU6fvAtaLta8mqybVLjSXXyx/IjRZxYQHI7VtM2zPvIR31hzHv+/3vjDev9Yct7vNUuQNhQQF4PnfdQcAvHpHL41bQ1ry+38dt/RqiX/c01vrZnhO4y8Jcn1JUXLQKvtSmYJH15atez9f5R4Rm6dGNi1nwijG1FY//XFfbqOm8h7JrSsVP8LF8glExEBEUcVXm76+Qn1ad1ba54g0vjXOhmZs1CqPbzQtantETl28guP5JaqcU1qY75Zevl+eHADu6le3ivTv3t2EKxXVHv9udY0Fu7MLxdvvTfCdpQaIlMBAREGVNRb3O3lJ6zVh5Bq3ddZrIcfRb+vTSoaj6JO0ZszrPx1W5ZxT/1uXqBrmIyvuuhMTFmzXk/HRpkwXe9c5c7kMnf76s3j7n+P6+mzdFSK5MBCB9gmgRhJgV0ek8Zz1eEi/STaWLy8qJg0EyirVWfzuWH5dwmZiVKgq59QDaU/GW6uOodSDXpF/rLAvhDa2t3/WDSHyBgMRg1ErZmoZ7fiCI9f0XWeKrlYhs4nFutS6QGvlmZuteRrbMi/h1IXGzerwRu821sXgbuqe5JOl3Z0JDQ7E+5JgpOeclS73LyyrxP/2nhNvX98pnkmqRB7gvxJoP9zhDbWaOuKaRDw5sjM+mmg/7VCJtWbqy7zYtIvrTd19uwrjjV3qhgxWHMxT9FxXK2vEwHDqDb5byMyZ0T3se9dSZi132DNSWW1B35dX2d0359buiraNyFcoFoi89tprGDx4MMLDwxEbG6vUafyOdPrudR2VW+beZDLhyZFdGpRWDlAhdD1dUOZRN7gj6R3iEBPm23UuUuLrirUVX23c6+SpX49fQEl5NaJDg8SFy/xJQIAJiycPsLuv55yVYvXV6hoLThdcQZe//Wy3z86/jUTnpCjV2klkZIoVW6isrMS9996L9PR0fPTRR0qdRhbG6Q+x9ohs/+sIZF0sw8D26i8CZTc0o9A5XvrhUKN/V41ASWtRkoJii37LxKwx3RQ718Of7AIAXJvSHOEhvlmbxZ2hXROx5I9puP//bRPve37pAezNKcTu05dxqt5Q4vRhnRDvg0sMEClFsU+Wl156CQCwePFipU7hl0ywJgxqlTQYqPNhLDWGjvRg0uAULN6c5XYadFNUVtfN+vLl2iyeGNwpHl8/mo57P9gi3mdbe0fqt1nD0TpWvdWwiXyBrr4/VlRUoLi42O5HDYa6dmndVumsGUO9cL7ljtTWAIAEBb95F16tW302JY5rfgxIaY6DL40Sk4Xr2/7cCAYhRI2gq0Bk7ty5iImJEX+Sk/17tUo9ipXkX+gxDvGXHhFbhdVzReW4fKXpy9U7UlhWV5Bv2jDfXLvHWxHmIEwf3hlHXhmNMT1bIC4iBDv/NhJZ88Yi0clMMyJyzatAZNasWTCZTC5/jhw50ujGzJ49G0VFReJPTk5Oo4/lDa3Xb/GG1m39v/v6anp+d/wkDkF8ZIi4/fbqY4qcI6/IWka+U2IkUts2U+QcRhUaHIj5D/THrudvYj4IURN5lSPy9NNPY9KkSS736dCh8VP8zGYzzGb+o3ZF6wtte50vy+0vPSLmoLrCZko85wslFXho4XYAdWXliYiU4FUgkpCQgISEBKXaohkjXbv01FQ9tcXGn1YOH94tEWuPnFdkKYF/SnpZdmRdkv34REQ2iuWIZGdnIyMjA9nZ2aipqUFGRgYyMjJQWqp8JUhfxgRRd/zn9emcGAkAWLItG1dlriZ7SZJ38sodPWU9NhGRlGKByAsvvIDU1FTMmTMHpaWlSE1NRWpqKnbu3On+l1VmpEsX4xDX/KlH5Maudb2TcvdaSAvK+fIigkSkPcUCkcWLF0MQhAY/Q4cOVeqUfkFX11kdRkU6bJJiBneMF3N2vFmm3h1BEPDr8Yviba4eS0RK0tX0Xc24uHi1iwt3/qCf0+M1X+tZRWqzBV4fb8mS7Zjf7T4rbv/x+vayHZeIyBEGIm70ah2jdRPs+NM3/sbwt9fn1AVrefGtp+QbmtmdfVnc7tKC66UQkbIYiMB12XIlS2g3jn6utHq86PvL9F1HcmQqw15dU/c336OV/y10R0TqYiACIC7SjN/1bunwsRqLvgIRP77OesbPXp93x6eK2ysO5MlyzHNFVwEA4we2RY9W+uoRJCLfw0Ck1r/v7wdzUMOXQ2dxiK6us3rMx/C3HpEOCXUF5i6VNb3U+/nicjFR9dY+joNzIiI5MRCRuL5TfIP7LDqLRFhHxLVm4cHud/Ih0h6L+etPNvl4/1pzXNxObsZEbSJSHgMRiTfv7dNgZc0aneWIKBmGDEixricysH1zz9qiw5hoxk2OV0b1F7tOX3a/kwufbcsWt1vEsLQ7ESmPgYhEs4gQTB/e2e4+veWIKNmaBQ9eixd+1x0fPNC/Scf5z4NN+/2miA0Pcb+Tj3nx1u7i9t3zNzf6OCfOlyCotiLcwPbNERzIjwciUh4/adwovlrlficVCQr20DSPCMEfrm+P5hGeXcy7JDme2nlzjxbImjcW13WKk7N55MSk6+xrfZRXNa7c++rD51FtERAfGYIvHx4kR9OIiNzyatE7f7T3TJHWTbCjh/6ZfS/ejPKqGhRfdV3Ns01sOIACdRrl5wa2b47tmdZaInlF5UhpxCrJpeXW93NMz5bMRSIi1bBHxGD0kLISHRqMxCj3+QO8lqlI8nfx+fZs5/u58O91JwAA4SEs6U5E6mEgYjg6iEQ8xEBEPaN7thC3F2w85XVu0ze7zojbB88Vy9YuIiJ3GIgYgPSCroceEZv6gUZ6h/o5IYxE1PJQejt0k5RjX7rnrIu9G3rm673idndWUyUiFTEQcSM0WPuXSFqCXmeTeOyE1evSZ4+IeoICA/DmvX3E26culnr8uxdKKuxuTx/eSbZ2ERG5o/1VVscmpLXFN48O1nzhuzDJMuxKzpqRG+MQdbWKDRO331vneXGzAa+tFrfv7tcG0aH+VRSOiLTFQMSF6zrFo2frGHygYV2M1rFh+GjSAM3Or1ev3N5D6yboTv1p155UBd5UW87dZuaorrK2iYjIHQYiLujhG/1vs4ajX9tY8bae+0Pqv15KDs1EmDnz3JENM4eK2/9cfczt/p9tO213m9VUiUhtDEQMQLqQm4FGZrA3R7kaLMw/caxdXF39kHfWnsChc8X4YMNJHM8vcbi/3ioHE5H/YSDiAa2veXazZnTcJ3Jj1wS72/vPKhiIaP6u6Je0RP8t7/yKeT8fwU1vb3S47y+H8sXt9c8MVbppREQNsH/bAEz2kYgu/f3uXrinf7Jq52OPiHOeltYvq7SvjNuYaqxERE3FHhEP6Omip6eedOnLMrpnSwQGqPdCsQS5c1GhwXj0xo4u93lt+SF0f2GleHtUjySlm0VE5BADEYPR89CMmhiGuDZrTLcG91XVWJBXVI7tmZfw4a+Zdo+9fmcvtZpGRGSHgYgH9JSPoNdkVU87KF66TZ5pt+wQcW94t0S7209+kYFBc9fgvgVb7O7/bdZwxEWa1WwaEZGIgYjB6CkOaUxbJg5OwbRhrocNPKGn4FCv3h7X1+728v25DvdrLSmERkSkNgYiHtDTt2+9Vlb15iUKDWr66q56ek/0KibMfYXUX/8yTIWWEBE5x0DEYPSUpNnYlsgRSunnVdC3JVPTnK6XtPv5m5DcPFzlFhER2WMgYhD39G+DXq1jMLijZ1MzfZ2O4jFdG9wxHltmjRBvj+nZAr/+ZRhOvX5Lg5LwRERaYB0RF2wXOz1c86Qrq+qR+j01enhXjKFZRAiy5o3VuhlERA6xR4QMScWSJUREpCAGIp7gRU9WcuTb6ilXhoiIGo+BCMmCAzNERNQYDEQ8wJoV8pKjM4MdIkREvoGBCMlC7cCAgQgRkW9gIOIBpS96r9zRE/GRxptK2dg8DVlyRNhLRUTkExiIuBAWos7s5gcHtdPtGjK6xTiEiMgnsI6IA7PHdMPh3GLc0CkegLLXvJYxoQD0tYZMY7CHgoiIGkOxHpGsrCxMmTIF7du3R1hYGDp27Ig5c+agsrJSqVPK5pEbO+Kfv09FgArFKpKbWUtsW/ygSyRIxteTYQ8RkW9QrEfkyJEjsFgsWLBgATp16oQDBw5g6tSpuHLlCt58802lTmtYfhCH4Ic/Xw8AEAzf/0NERHJRLBAZPXo0Ro8eLd7u0KEDjh49ivnz5xsuEFG0eFbtoY3eI+LJS3RNy2gZz8c+ESIiX6BqsmpRURGaN2+u5il1z3Y5NXgc4hXmkxARkY1qyaonTpzAu+++67I3pKKiAhUVFeLt4uJiNZrmlpKXTdsXe8HHI5HHR3TWuglERKRDXveIzJo1CyaTyeXPkSNH7H7n7NmzGD16NO69915MnTrV6bHnzp2LmJgY8Sc5Odn7Z6SRTc8Oa9Tv2XoHLAaMQ7wJ0CYNThG35cgRYZ8KEZFv8LpH5Omnn8akSZNc7tOhQwdx+9y5cxg2bBgGDx6M//znPy5/b/bs2ZgxY4Z4u7i4WBfBiCfpCG1qZ7809ti+nsDJwIGIiBzxOhBJSEhAQkKCR/uePXsWw4YNQ//+/bFo0SIEBLjugDGbzTCbzd42ydBMYrKqtu1oKnfBmty5pZXVFnkPSEREmlAsWfXs2bMYOnQo2rZtizfffBMXLlxAXl4e8vLylDqlYlRJrjR4IOLIg4Paidtyv4ZBgexjISLyBYolq65atQonTpzAiRMn0KZNG7vHfD0x0xt1OSK+95o8NrQjPtl62npD5rihdWyYvAckIiJNKNYjMmnSJAiC4PCH6ph8pI6II9LhGLmHZgQAKhS+JSIihXHRO09ILnhyFuWS8r0wRHkBLGpGRGR4DESaoH18hGzHMnqHiKMcEOl9cj8/E+TvZSEiIvUxEPGA9IInHVqSY5jJUanyvsmxTT6u2hxNP3YWKMgRlAiwD3TiI0NwZ2rrph+YiIhUpVplVV8kx5d8fqkHggNNqKppxKspefG2PTcSgUwaISIyHPaIeEB6eZN+m5fjm70vDy9IczhcPc/QoECvj22CfbIqgxAiImNij4iXereJwdH8EgDyzHQx8uXTXRCVEGXGnamtEWAyITo02Ol+LWNDUZJf6v35Df3qERERwEDEI9I8jkdu7IhOiZEY2jURf1i8w+Nj3N2vDczBAViyLdvpsW2MkrfaIiZU3A52UjX37XF93R7ngwf6Y87/DuLX4xe9Or8v9yYREfkLBiJeMgcF4JEbOzbqd8ODnQ9BdEiIwKkLVxAT5rznQG/MQYHY9+LNCDSZENCEoZEOCZH4ZEoaUmYt9/h3rMmqRERkdMwRaQJvZ804uljb7lk0aQDu6d8G3z6WLkPL1BMdGowIszbxLOuIEBEZH3tEPODscuftQnWOLpy2u9rFReDNe/t4d0AfE2kOQmlFtUf7msT/EBGRkbFHpAkc1c5wJdDhq+1/V1Nnr9rKp4Z4dQz2iBARGR8DEQ/IUZhLgOCyR6TRB/Yh3i5kxziEiMj4GIg0gbfhgsNARJ6m+CW+dkRExsdApAm8Tlb1tEfEjy2ZmubxvhyaISIyPgYiTfDEiM4AgLv6ebbGieMckYb8aWDG9hraDO4Y7/HvMg4hIjI+zpppggcGtcP1nRPQrnm4+50FZ9N3/ftq+tRNXRr9u46KwRERkbEwEGkCk8mE9vERHu/PoRl58aUjIjI+Ds2oKMhBj4ijQCYxKrTBfdQQgzgiIuNjj4iXmnLxc9QjMn14pwb3vXpHT9RYLJg4OKXxJ9MzmaYn+/uwFhGRL2CPiIzeHZ+KLkmRmJDW1uHj9TtE/jb2GoSHNIwFW8SEYtHkgRjaNVGJZvqEkMCABq8nEREZD3tEZHRrn1a4tU8rLNyU2eAxAUBgvSvnlOvbq9QyY7mhc7zTlXjH9m6JuIgQpMRHMFmViMgHsEdERXGRZrvbfnshdfO859zaw+ljT43sgpdv7wkACA9xvpoxEREZAwMRlbSLC8foHi20boY+uMkRCQ70LEB7Z3wq2sdH4N3xqXK0ioiINMChGQ9Ih1Qc5XS48sXDg/DLwXw8emNHh3VEqKEgDyu/XdMyGuueGapsY4iISFEMRDxgDgrE2+P6oLLaguYRIV797qAOcRjUIU6hlvmmYAZsRER+g4GIh+5MbaN1E/xG/aRee/5UAJ+IyPcxR4RU5y6UcDU0Ex0aLG9jiIhIU+wRId1xlKzaPCIEf73lGiRGs+osEZEvYY8Iqc5dBkhQQMM/y1E9WuDu/hweIyLyNQxESHXuhmY8nb5LRETGx0CEdMdvC70REfkhBiJkCIxNiIh8EwMRlX3x8CB0iI/AZ39M07opmkliwikREdXirBmVDeoQh7V+Xg103IBkHM0rwQ2d47VuChERaYyBCKkuODAAr9zRU+tmEBGRDnBohgwhLJgr7RIR+SJFA5HbbrsNbdu2RWhoKFq2bIkHH3wQ586dU/KU5GOCA03o3SYG04d10ropRESkAEUDkWHDhuGrr77C0aNH8e233+LkyZO45557lDwl+ZgHBrXD/6Zfj2ZeLjZIRETGoGiOyFNPPSVut2vXDrNmzcIdd9yBqqoqBAdzzRByT+Aad0REPk21HJFLly7hs88+w+DBgxmEkMcERiJERD5N8UDk2WefRUREBOLi4pCdnY1ly5Y53beiogLFxcV2P+TfurSI0roJRESkIK8DkVmzZsFkMrn8OXLkiLj/zJkzsWfPHvzyyy8IDAzEQw895PRb7ty5cxETEyP+JCcnN/6ZkaH9b/p1mD2mG8Zdy78BIiJfZhK87Pu+cOECCgoKXO7ToUMHhIQ0TC48c+YMkpOTsXnzZqSnpzd4vKKiAhUVFeLt4uJiJCcno6ioCNHR0d40U1MLN2Xi5R8PAQCy5o3VuDVERETqKi4uRkxMjEfXb6+TVRMSEpCQkNCohlksFgCwCzakzGYzzGZzo45NRERExqPYrJlt27Zhx44duP7669GsWTOcPHkSzz//PDp27OiwN4SIiIj8j2LJquHh4fjuu+8wYsQIdO3aFVOmTEHv3r2xYcMG9noQERERAAV7RHr16oW1a9cqdXgiIiLyAVxrhoiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0BEAV2SorRuAhERkSEEad0AX3R953i8dV8fdE5kQEJEROQKAxGF3NWvjdZNICIi0j0OzRAREZFmGIgQERGRZhiIEBERkWYYiBAREZFmGIgQERGRZhiIEBERkWYYiBAREZFmGIgQERGRZhiIEBERkWYYiBAREZFmGIgQERGRZhiIEBERkWYYiBAREZFmdL36riAIAIDi4mKNW0JERESesl23bddxV3QdiJSUlAAAkpOTNW4JEREReaukpAQxMTEu9zEJnoQrGrFYLDh37hyioqJgMplkPXZxcTGSk5ORk5OD6OhoWY+tNV9+bgCfn9H58vPz5ecG8PkZmdrPTRAElJSUoFWrVggIcJ0FousekYCAALRp00bRc0RHR/vcH5yNLz83gM/P6Hz5+fnycwP4/IxMzefmrifEhsmqREREpBkGIkRERKQZvw1EzGYz5syZA7PZrHVTZOfLzw3g8zM6X35+vvzcAD4/I9Pzc9N1sioRERH5Nr/tESEiIiLtMRAhIiIizTAQISIiIs0wECEiIiLN+GUg8t577yElJQWhoaFIS0vD9u3btW6SW3PnzsWAAQMQFRWFxMRE3HHHHTh69KjdPkOHDoXJZLL7efTRR+32yc7OxtixYxEeHo7ExETMnDkT1dXVaj4Vh1588cUGbe/WrZv4eHl5OaZNm4a4uDhERkbi7rvvRn5+vt0x9PrcACAlJaXB8zOZTJg2bRoA4713GzduxK233opWrVrBZDJh6dKldo8LgoAXXngBLVu2RFhYGEaOHInjx4/b7XPp0iVMmDAB0dHRiI2NxZQpU1BaWmq3z759+3DDDTcgNDQUycnJ+Mc//qH0U3P53KqqqvDss8+iV69eiIiIQKtWrfDQQw/h3Llzdsdw9H7PmzdP8+cGuH/vJk2a1KDto0ePtttHr+8d4P75Ofp3aDKZ8MYbb4j76PX98+Q6INdn5fr169GvXz+YzWZ06tQJixcvVu6JCX7miy++EEJCQoSFCxcKBw8eFKZOnSrExsYK+fn5WjfNpVGjRgmLFi0SDhw4IGRkZAi33HKL0LZtW6G0tFTc58YbbxSmTp0q5Obmij9FRUXi49XV1ULPnj2FkSNHCnv27BF++uknIT4+Xpg9e7YWT8nOnDlzhB49eti1/cKFC+Ljjz76qJCcnCysWbNG2LlzpzBo0CBh8ODB4uN6fm6CIAjnz5+3e26rVq0SAAjr1q0TBMF4791PP/0k/PWvfxW+++47AYDw/fff2z0+b948ISYmRli6dKmwd+9e4bbbbhPat28vXL16Vdxn9OjRQp8+fYStW7cKv/76q9CpUydh/Pjx4uNFRUVCUlKSMGHCBOHAgQPC559/LoSFhQkLFizQ7LkVFhYKI0eOFL788kvhyJEjwpYtW4SBAwcK/fv3tztGu3bthJdfftnu/ZT+W9Xqubl7foIgCBMnThRGjx5t1/ZLly7Z7aPX986T5yd9Xrm5ucLChQsFk8kknDx5UtxHr++fJ9cBOT4rT506JYSHhwszZswQDh06JLz77rtCYGCgsGLFCkWel98FIgMHDhSmTZsm3q6pqRFatWolzJ07V8NWee/8+fMCAGHDhg3ifTfeeKPwxBNPOP2dn376SQgICBDy8vLE++bPny9ER0cLFRUVSjbXrTlz5gh9+vRx+FhhYaEQHBwsfP311+J9hw8fFgAIW7ZsEQRB38/NkSeeeELo2LGjYLFYBEEw9ntX/8PeYrEILVq0EN544w3xvsLCQsFsNguff/65IAiCcOjQIQGAsGPHDnGfn3/+WTCZTMLZs2cFQRCE999/X2jWrJnd83v22WeFrl27KvyM6ji6kNW3fft2AYBw+vRp8b527doJb7/9ttPf0cNzEwTHz2/ixInC7bff7vR3jPLeCYJn79/tt98uDB8+3O4+o7x/9a8Dcn1W/uUvfxF69Ohhd65x48YJo0aNUuR5+NXQTGVlJXbt2oWRI0eK9wUEBGDkyJHYsmWLhi3zXlFREQCgefPmdvd/9tlniI+PR8+ePTF79myUlZWJj23ZsgW9evVCUlKSeN+oUaNQXFyMgwcPqtNwF44fP45WrVqhQ4cOmDBhArKzswEAu3btQlVVld371q1bN7Rt21Z83/T+3KQqKyvx6aef4g9/+IPdYo5Gfu+kMjMzkZeXZ/d+xcTEIC0tze79io2NxbXXXivuM3LkSAQEBGDbtm3iPkOGDEFISIi4z6hRo3D06FFcvnxZpWfjXlFREUwmE2JjY+3unzdvHuLi4pCamoo33njDrutb789t/fr1SExMRNeuXfHYY4+hoKBAfMyX3rv8/HwsX74cU6ZMafCYEd6/+tcBuT4rt2zZYncM2z5KXSd1veid3C5evIiamhq7NwAAkpKScOTIEY1a5T2LxYInn3wS1113HXr27Cnef//996Ndu3Zo1aoV9u3bh2effRZHjx7Fd999BwDIy8tz+Nxtj2kpLS0NixcvRteuXZGbm4uXXnoJN9xwAw4cOIC8vDyEhIQ0+KBPSkoS263n51bf0qVLUVhYiEmTJon3Gfm9q8/WHkftlb5fiYmJdo8HBQWhefPmdvu0b9++wTFsjzVr1kyR9nujvLwczz77LMaPH2+3kNjjjz+Ofv36oXnz5ti8eTNmz56N3NxcvPXWWwD0/dxGjx6Nu+66C+3bt8fJkyfx3HPPYcyYMdiyZQsCAwN95r0DgI8//hhRUVG466677O43wvvn6Dog12els32Ki4tx9epVhIWFyfpc/CoQ8RXTpk3DgQMHsGnTJrv7H374YXG7V69eaNmyJUaMGIGTJ0+iY8eOajfTK2PGjBG3e/fujbS0NLRr1w5fffWV7H/0Wvvoo48wZswYtGrVSrzPyO+dv6qqqsJ9990HQRAwf/58u8dmzJghbvfu3RshISF45JFHMHfuXF2W2Jb6/e9/L2736tULvXv3RseOHbF+/XqMGDFCw5bJb+HChZgwYQJCQ0Pt7jfC++fsOmBEfjU0Ex8fj8DAwAYZxPn5+WjRooVGrfLO9OnT8eOPP2LdunVo06aNy33T0tIAACdOnAAAtGjRwuFztz2mJ7GxsejSpQtOnDiBFi1aoLKyEoWFhXb7SN83ozy306dPY/Xq1fjjH//ocj8jv3e29rj6d9aiRQucP3/e7vHq6mpcunTJEO+pLQg5ffo0Vq1a5XZZ9bS0NFRXVyMrKwuAvp9bfR06dEB8fLzd36KR3zubX3/9FUePHnX7bxHQ3/vn7Dog12els32io6MV+WLoV4FISEgI+vfvjzVr1oj3WSwWrFmzBunp6Rq2zD1BEDB9+nR8//33WLt2bYNuQUcyMjIAAC1btgQApKenY//+/XYfIrYP0e7duyvS7sYqLS3FyZMn0bJlS/Tv3x/BwcF279vRo0eRnZ0tvm9GeW6LFi1CYmIixo4d63I/I7937du3R4sWLezer+LiYmzbts3u/SosLMSuXbvEfdauXQuLxSIGYenp6di4cSOqqqrEfVatWoWuXbtq2rVvC0KOHz+O1atXIy4uzu3vZGRkICAgQBzS0Otzc+TMmTMoKCiw+1s06nsn9dFHH6F///7o06eP23318v65uw7I9VmZnp5udwzbPopdJxVJgdWxL774QjCbzcLixYuFQ4cOCQ8//LAQGxtrl0GsR4899pgQExMjrF+/3m5KWVlZmSAIgnDixAnh5ZdfFnbu3ClkZmYKy5YtEzp06CAMGTJEPIZt2tbNN98sZGRkCCtWrBASEhJ0McX16aefFtavXy9kZmYKv/32mzBy5EghPj5eOH/+vCAI1ilpbdu2FdauXSvs3LlTSE9PF9LT08Xf1/Nzs6mpqRHatm0rPPvss3b3G/G9KykpEfbs2SPs2bNHACC89dZbwp49e8SZI/PmzRNiY2OFZcuWCfv27RNuv/12h9N3U1NThW3btgmbNm0SOnfubDcFtLCwUEhKShIefPBB4cCBA8IXX3whhIeHKz5F0tVzq6ysFG677TahTZs2QkZGht2/RduMg82bNwtvv/22kJGRIZw8eVL49NNPhYSEBOGhhx7S/Lm5e34lJSXCM888I2zZskXIzMwUVq9eLfTr10/o3LmzUF5eLh5Dr++du+dnU1RUJISHhwvz589v8Pt6fv/cXQcEQZ7PStv03ZkzZwqHDx8W3nvvPU7fldu7774rtG3bVggJCREGDhwobN26VesmuQXA4c+iRYsEQRCE7OxsYciQIULz5s0Fs9ksdOrUSZg5c6ZdLQpBEISsrCxhzJgxQlhYmBAfHy88/fTTQlVVlQbPyN64ceOEli1bCiEhIULr1q2FcePGCSdOnBAfv3r1qvCnP/1JaNasmRAeHi7ceeedQm5urt0x9PrcbFauXCkAEI4ePWp3vxHfu3Xr1jn8e5w4caIgCNYpvM8//7yQlJQkmM1mYcSIEQ2ed0FBgTB+/HghMjJSiI6OFiZPniyUlJTY7bN3717h+uuvF8xms9C6dWth3rx5mj63zMxMp/8WbTVhdu3aJaSlpQkxMTFCaGiocM011wivv/663YVcq+fm7vmVlZUJN998s5CQkCAEBwcL7dq1E6ZOndrgi5pe3zt3z89mwYIFQlhYmFBYWNjg9/X8/rm7DgiCfJ+V69atE/r27SuEhIQIHTp0sDuH3Ey1T46IiIhIdX6VI0JERET6wkCEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTz/wGH+oXLwDrbCwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.plot(total)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb368900-34b3-4c36-a916-62ed3a97173b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50253cc9-254c-4933-94bd-36c28d0f97b0", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "895a5767-2865-477f-aa21-094a711bf033", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/hyena_test/simple_hyena_model.ipynb b/hyena_test/simple_hyena_model.ipynb index 2088487..966ab89 100644 --- a/hyena_test/simple_hyena_model.ipynb +++ b/hyena_test/simple_hyena_model.ipynb @@ -343,7 +343,9 @@ " # Increase the channel dimension from 1 to d_model\n", " u = self.fc_before(u) \n", " # Pass through the operator\n", + " #[B,1024,128] --> [B,1024,128]\n", " u = self.operator(u)\n", + " \n", " last_state = u[:,-1,:]\n", " # Decrease the channel dimension back to 1\n", " y = self.fc_after(last_state)\n", @@ -564,10 +566,127 @@ "r2_scores = fit_and_evaluate_linear_regression(outputs_and_params)" ] }, + { + "cell_type": "markdown", + "id": "de65d0d2-b0c6-4ac5-a87f-70fa7f90480b", + "metadata": {}, + "source": [ + "# Autoregressive" + ] + }, + { + "cell_type": "code", + "execution_count": 120, + "id": "8ee139a6-aee5-4309-8685-bf0c28893279", + "metadata": {}, + "outputs": [], + "source": [ + "def predict_autoregressive(model, initial_data, n_steps):\n", + " model.eval()\n", + " predictions = []\n", + " current_input = initial_data\n", + " with torch.no_grad():\n", + " for _ in range(n_steps):\n", + " # Get the prediction for the next step and save it\n", + " next_output, last_state = model(current_input)\n", + " predictions.append(next_output)\n", + " #import pdb;pdb.set_trace()\n", + "\n", + " # Prepare the input for the next step\n", + " next_input = torch.cat((current_input[:, 1:, :], next_output[:, None,:]), dim=1)\n", + "\n", + " current_input = next_input\n", + "\n", + " return torch.cat(predictions, dim=1)" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "61c7e1db-ffee-4ef2-a8c6-5756e326904c", + "id": "272040d6-4dfb-438b-a432-744e950effd1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 167, + "id": "9f49cbf8-08e8-428a-af80-bcb2515a6327", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 1024, 1])" + ] + }, + "execution_count": 167, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "initial_data = dataset[0]\n", + "initial_data[0][None,...].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 168, + "id": "c75464e3-e44d-4325-8804-29d8081e3a45", + "metadata": {}, + "outputs": [], + "source": [ + "autoregressive_out = predict_autoregressive(model,initial_data[0][None,...],500)" + ] + }, + { + "cell_type": "code", + "execution_count": 169, + "id": "90a37c56-59f3-49bc-bfbe-d9e126c42ed1", + "metadata": {}, + "outputs": [], + "source": [ + "total = np.concatenate([initial_data[0].squeeze().numpy() ,autoregressive_out.squeeze().numpy()])" + ] + }, + { + "cell_type": "code", + "execution_count": 170, + "id": "3d2ad0ee-7a0e-4b6f-8b40-dd565c8af84d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 170, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABhFElEQVR4nO3deXhU1fkH8O8syWTfd0ggQCBA2JcA4koUrFq3alUqgorVitXqz4W2aqu1WOtuXWtxqbstbqhQZBGRyB4gLGEnISEb2bdZz++PZCYzyWxJZubemfl+nicPZO6dyXshM/POOe95j0IIIUBERETkJ5RSB0BERETUF0xeiIiIyK8weSEiIiK/wuSFiIiI/AqTFyIiIvIrTF6IiIjIrzB5ISIiIr/C5IWIiIj8ilrqADzNZDKhoqIC0dHRUCgUUodDREREbhBCoLm5GRkZGVAqnY+tBFzyUlFRgczMTKnDICIion4oKyvD4MGDnZ4TcMlLdHQ0gM6Lj4mJkTgaIiIickdTUxMyMzMt7+POBFzyYp4qiomJYfJCRETkZ9wp+WDBLhEREfkVJi9ERETkV5i8EBERkV9h8kJERER+hckLERER+RUmL0RERORXmLwQERGRX2HyQkRERH6FyQsRERH5FSYvRERE5FeYvBAREZFfYfJCREREfoXJCxEREbmlQ2/E0hV78J8dpySNI+B2lSYiIiLPq2vVYeFbW7HnVCO+LKrAnNwUxEeGShILkxciIiJyqqZZi+veKMTRmlbER4TgpesnS5a4AExeiIiIyIl2nRG3vrsdR2takREbhndvyceIlChJY2LyQkRERHYJIfDQij3YXdaAuIgQvL94BrKTIqUOiwW7REREZN/nReX4oqgCKqUCr/9qiiwSF4DJCxEREdlRVteGRz7fBwC4Z04O8oclShxRNyYvREREZEMIgYe/KEaz1oApQ+Jxx3nDpQ7JBpMXIiIisrH2QDU2lNQgRKXA338xHmqVvNIFeUVDREREkurQG/HYyv0AgFtmD8OwZGlXFtnD5IWIiIgs/l14EqV1bUiN0eCuC0ZIHY5dTF6IiIgIANCiNeDV748CAO67cBQiNfLsqMLkhYiIiAAAb/94HHWtOmQnReKqyYOkDschJi9ERESEpg493th4DABwT0GO7Ip0rck3MiIiIvKZj7aWoqnDgOHJkbh0fIbU4TjF5IWIiCjI6QwmLN90AgDw63OGQ6VUSBuQC0xeiIiIgtzXeytQ2dSBpCgNLp8k71EXgMkLERFRUBNC4I2NxwEAi84aCo1aJXFErjF5ISIiCmJbjtfhwOkmhIeoMD8/S+pw3MLkhYiIKIh9sKUUAHDFpEGIiwiVOBr3MHkhIiIKUnWtOqwqrgQAvxl1AZi8EBERBa3/7CiDzmjCuEGxyBsUK3U4bmPyQkREFISEEPhwaxkA4AY/GnUBmLwQEREFpcKjZ3C8thWRoSr8fIL8l0db82rysnHjRlx22WXIyMiAQqHA559/7vI+GzZswOTJk6HRaDBixAi8/fbb3gyRiIgoKH2yvXPU5fJJg2S7AaMjXk1eWltbMWHCBLz88stunX/8+HFccsklOP/881FUVIR77rkHt956K1avXu3NMImIiIJKq9aA1fuqAADXTBkscTR959VU6+KLL8bFF1/s9vmvvfYasrOz8cwzzwAARo8ejU2bNuG5557D3LlzvRUmERFRUPnf/kq0640YmhiBiZlxUofTZ7KqeSksLERBQYHNbXPnzkVhYaFEEREREQWez3ZVAOjs7aJQyHsfI3tkNclVWVmJ1NRUm9tSU1PR1NSE9vZ2hIeH97qPVquFVqu1fN/U1OT1OImIiPxVdXMHNh2uAQBcMXGQxNH0j6xGXvpj2bJliI2NtXxlZmZKHRIREZFsfbX7NEwCmJgZh6FJkVKH0y+ySl7S0tJQVVVlc1tVVRViYmLsjroAwNKlS9HY2Gj5Kisr80WoREREfunzXeUAgCsn+eeoCyCzaaOZM2fim2++sbltzZo1mDlzpsP7aDQaaDQab4dGRETk90rPtGFveSOUCuCS8elSh9NvXh15aWlpQVFREYqKigB0LoUuKipCaWnnJlBLly7FggULLOfffvvtOHbsGB544AEcPHgQr7zyCj755BP87ne/82aYREREQeHb4tMAgBnDEpEU5b8f/L2avGzfvh2TJk3CpEmTAAD33nsvJk2ahEceeQQAcPr0aUsiAwDZ2dn4+uuvsWbNGkyYMAHPPPMM3nzzTS6TJiIi8oBvuzZhvDgvTeJIBkYhhBBSB+FJTU1NiI2NRWNjI2JiYqQOh4iISBYqGtox68l1UCiALUvnICUmTOqQbPTl/VtWBbtERETkHau6Rl2mDomXXeLSV0xeiIiIgoA5eZmX57+FumZMXoiIiAJcdXMHtp2sAwDM8/N6F4DJCxERUcBbva8KQgATMuMwKM5+3zR/wuSFiIgowK3Z39kAdt5Y/x91AZi8EBERBbRWrQE/HT0DALhwTIrE0XgGkxciIqIAtulILXRGE7ISIjA8OUrqcDyCyQsREVEAW3egGgAwZ3QKFAqFxNF4BpMXIiKiAGUyCawr6UpeclMljsZzmLwQEREFqOKKRtQ0axEZqsL07ASpw/EYJi9EREQBam3XlNE5I5MRqg6ct/zAuRIiIiKyse5gZ/JyQW5grDIyY/JCREQUgKqaOrC3vBEKBXA+kxciIhqI7SfqcMHTG7DxUA0AYENJNT7dXiZxVBRovu/6/Ro/OA5JURqJo/EsJi9ERD52/T9/wrHaVixYvhVagxEL39qG+/+zB6cb26UOjQLID4drAQDnjkyWOBLPY/JCRORjeqOw/P3NH45b/q7VmyCEsHcXoj4xmQQ2He4ceTknJ0niaDyPyQsRkYRqW7SWvz/wnz045+/r0ao1SBgRBYJ9FU2ob9MjWqPGhMw4qcPxOCYvREQ+YjL1HlUxWt229UQdyurasXJPhS/DogC0sWvUZebwRISoAu+tPvCuiPrNaBJo09l+4uMQNpFnPPu/Ekz5yxqU1bXZ3G6wk9CYb2rXGX0RGgWgH7qSl7MDsN4FYPJCVq5+dTPGPLIaZ7qGsc+0aHH+0xtw14e7ep37v32V+HfhCR9HSOS/Xlx3BPVtejy35pDN7R9sKbV7/svrj2D0I6uw9kCVL8KjANKqNWDHyXoAgVnvAjB5IStFZQ0AupsaPbvmEE6cacNXu3sPYd/27x14+It9luyeiLq164w4Udtq/6Ab++IJAfx9dQkA4JZ3tuOCpzdgy7EzHoyQAtlPx85AbxTISojAkMRIqcPxCiYv1It5EPtwVYvLc2/819Zew+DW2nVGmzl9omBw6Us/4LynN1g+/VpT9mNX32O1rfjlGz/h1ne2Wz5k1LfqcOs727Cq+PRAw6UAY14ifXaAjroATF7Inq5co9RBUqI3mmy+7zkMbtbYpsfoR1bh6lc3ezQ8Irk7WtM56mJv1LLvqUu37w5U4YqXfwQAPLOmBN8dqMbt7+0cwCNSIDIX656dE5j1LgCTFwKgM5jw8bbueXfRlb3Utensnt+uty0iXLGr3GZpZ1ldGx79ohjvbTkJoHs6iigYlFQ2W/5uMJl6He/HwItdZ1rsPz8puJU3tONYTStUSgVmDk+UOhyvUUsdAEnvte+P4lmr0RPzAiOdofcLLwB06HuvgPjPjlO4adZQAMBNy7fimKP5fqIAt3TFHsvf7U2ZKtwYexFwPtWqM5j6Nf1EgW/zkc4po/GDYxEbHiJxNN7DkRfC+pJqm+8FgMrGDofnd+h6JzXWiQ4TFwpm1kufDcbeScjHbuxhZK8fjLXb39sBpZLJC/VW2FXYPXNY4I66AExegp7WYMSu0gab24QANh+ttbmtvKEdf/hsL45UN6PD0HvkpbKpo9d9iIKRdWuk/har6+wkPdbWHayGirkL9SCEwE9Hu5KXAJ4yAjhtFPS+2NW7oFBAoK7Vdj59wb+24GhNK9530JPiX5uO41+bjmPlXbPtHjeZBD8pUtAx9rPJozvbAzS06/v12BS4SuvaUNHYgRCVAlOGxEsdjldx5CXI2RtFEQKo71Gsa1494cpBq2JFa3o7hYtEgWhveaPl7/srmvr1GM86WMFnbUMJeyyRrZ+6powmDI5DRGhgj00weQlySVGaXrcJAPVt/ftUV91sv1bG3tw/UaA7XO26VxKRpxQGyZQRwOQlaOgMJqw/WN1rONpkb1hbCNS39m8Z5lOrSuzezuSFggEbMpJUhBBBU6wLMHkJGk+tOohFb2/DHe/vtKxkeGntYSz5oPe+RZ0jL57tIaEzctqIAp+9vi7c3JR84cSZNlQ1aRGqUmJygNe7AExegsYHWzsLbTceqsGkx9egpLIZzziYVxcCqG/1bDGgvRd1okBjb4SxXW/s1ZXa8z+Xz69gZ54ympgVh7AQlcTReB+TlyChtlrp09iux8NfFDs896V1h1FSZb/wtr8e+M8e1ycR+blDdp43b/5wHPd9sturP3fFznIAwLuFJ3DvJ0WcvgpCwTRlBDB5CRohKtv/ameLlmvdbDsepXG/mt28URhRoPp6z2lc+UrvfbyeXXMIX9rZ48iT9pQ3AAAe+WIfVuwsx9oDVV79eSQvQoigKtYFmLwEDVWPHiue6Cx+QW7KwB+EKADUtepw5wfSbZCYEBFq832LG31iKHAcrWlFbYsWGrUSEzPjpA7HJwJ7IThZqHskL+UN7QN+zL4OTNe2aLFydwX0RoHF5wwb8M8nkpoQAi1aA2qatR55vFCVsl/F7Wmx4R75+eSfzP1dJmfFB0W9C8CRl6Ch7jFtVFbngeSlj6soLntpE/701X488c0B7D3V6PoORDLzxsajuGn5VmgNRrRqDbjzg50Y96f/9dofzJWFXZuY9tTf0UylArj1nW2W780jq0IINHWwE2+g23aiDgCQPyxB4kh8h8lLkFB7YSOUvq4APW212eNl/9iEWcvWYvW+Sg9HReQ9f/3mIL4/VIMviiqw6K1t+GZv5+/vk98e7NPjzBqeiJ+WzsFvzhtuuW1+fhYeu3xsv+J6aMVefHegdwL124+KMP5P/8OeUw39elzyD9tP1AMApg9l8kIBpue0kTtumZ3t9PhAh8orGjvw63/vGNBjEEmhQ2/E1q5Pu/2hVimQFhuGSKui99+cPwLxkaFO7uW+dwtP4sZ/bcFXXYXCb2w85pHHJfkpb2hHeUM7VEoFJmbFSR2OzzB5CRJqZd//q13lO/14SKKAoBhgxbv5/taF9KEqZa9VgQBw/9xRfX78XaUNXOEXJLZ3JdF5GTEBv5+RNb79BImQfkwbudoF+s8/z8Pw5Eib284fldznn2OtpLIZb2w8Cq2dDSOJ5KI/qUtydPc+YuZmdtbbc4SqO1+OhyRGWG4bFBeOO87tnloi6sk8ZTQ1iKaMACYvAa+4vBE/HTuD3S4KZH99bu/VPyo7ny6tbxqVFo2vf3u2zfElF4zoX6Bd5j6/EX/95iD+yWFukjF3VuvNz8+y/P3ivDSb55PO0LmiKNRqpEXTlbxMs3oTCg9VufwQ4Q62rAtc5mLdaUMDf0sAa0xeAlhFQzsufWkTrnvjJ5fnThwch5V3zba5LTc9BgtmDrG5LTvRdqSl5zD3hMFxmOqBfTWKyrgaiaTV1KHHre9ss9tg7tUNR13e/4krx1n+bjQJmyXQ5u0CrIf5zYnMw5eOsdwWHiTLXql/Gtv1lm7oU4Zw5IX81BNf78fPXvgB7Toj2nVGbD3ufkHh7JwkxIaHWL6/ZXY2Lhufjscuz8O2PxRg3tg0vHvz9F7369n8Tq1S4j93zLK0qL5q0iCXP7u+VYfff7YXu0rrLbd5ooke0UC8vO4IvjtQjd9+2Ll56UA2WDQJ4JycJMv3Okvy0p2cmEdYrJ+H4V3Hn7xqHO4pyOn3z+fQS2DaWVoPIYDspEibaclgEDzVPUHgnz8cBwB8tacCT68uQXUfVgNFh4XAejuUhbOGWooKk6M1eO3GKQA6iwfveH+nzZC4Pf9aOBV7TzUiJjwEK3aVOz330S/34cvdFfhgS6nlNg+MlBMNSM/VdPsqmty636C48F5FtiYh8NgVefi8qHMUZ3BcZ1M5V1tsmEderpve+Xx7/rvDbsVAwcFcrOuJ0W5/w+QlABlNok+Ji1m01QtpUpT9LP7icenY+oc5SLY6PiEzDrvLGmzOiwhVI39YIo5Uu97gcbedHhSKfpVEEnmOscdIy6UvbXJ5n/n5WTbTRWEhSnToTZg9IgkxYSH4+LYZKKlqtuw/c87IZIwbFIuRqdF2H4/TRuTMtq5i3WlBVqwLMHkhK0qlAjv+WACjSViGq+1JiQ6z+f53BTlY+NY2XDt1cK9z3VlSevJMm537uREwkRf1Z2PmnjVg3917LrYcq8PlEzMAAPnDEpFvtetvqFqJr3rUmlmL6PE8TIoKdXvjVGuC80YBR2swWj40Tg2yYl2AyUtAsq4d6atEByMuzpw3KgU7H74Q8REhvY7ZW7HkDiYvJDXrZcz/2nTcrfv0bAY5OD4Cg6dEODjbtbAeycsPD1yAD7eW4rGV+/v9mBQYisuboDWYkBgZiuykSNd3CDBMXgLQJ9tP+fxnJjjoDNqzoNddnDYiqVkX6D7uZrLQqvNsf6K0GNtRzvBQFW6enY0Lx6SirlWHy1/+0aM/j/yHpd5laPyAmyb6I642Iq/q93Mq+J6LJDPGfswbuVPj5Y6nrh6PC3JTcOvZ9rfoyEyIwITMOMt0lCsDWChFMhXM9S4Ak5egMyY9xqc/r/8jL0TS6k/Ni7lL7kBdOy0TyxdOc9nu3dyp15VviytxtKbFE6GRDAghsOOkeeSFyQv5MXd7UIzN6J289CwK9CSlnaGXc0e63kIgGIdBSV7609el3cPTRq606gxun3vn+zu9GAn50rHaVtS36aFRK33+gVQumLwECHeGuJ/6xXik9phDB9Crs64n2Ute3ImVqQtJrT/TRu16k+uTPKi5w/3k5Ug1R14Cxa7SBgDA+MGxHhvt8zfBedUByODGC+2QhAib7p1AZ6HtsOQob4Vld9rIYHL9Aq9QAG/9eBwXPL0BK/dUWNqpE/mC0SSwvqSmz/eL9OIopj1N7XqHx3rudcOeMYHDvKJ0UlbwLZE2Y/ISINz5lKhSKnB9fhYWnTXUcpu3O9nae3x3R17+/NV+HKttxZIPduHp1SWeD47IgTX7K/t1vyevHuf6JA/q+WEEAJ65ZgLW/995GJFi2/iu57Jr8l/mkZdJmXGSxiElJi8Bwp2RF6CzHfmjl421fB8T1vvFz5Ps7Yhr3lHXmZ41L2/8wF2myXcanYxo9PTOzdPxm/OG4+hff9YrYfC2p34xHvnZtgWbV08ZjOykyF4jLT23OyD/1Ko14GBl51YVHHkhv9fX+fnXfjUZI1Ki8PL8yV6KqJO9mpcON+oCet6LSz3Jl/pSMH7uyGQ8MC+33yvrBmJYchQ+/vVMjBsU2+tYWEjvl/cTta2+CIu8aM+pRpgEkBEbhrTY3jWMwYLJS4Bwp47E+v1/Xl46vrv3XIz2cqW6vQ67WoPrFRlcbURSMBhNeHbNIWzrw47scjC7a8dq6w6/GnXvaaLvDlT5LCbyjl1lrHcB2GE3YPRnZYQvKO2kxx16E76//zw0tRtw2T/sb3b3352+7xJM9OG2Mry41v92br57Tg4SI0MxZ3Sq5TZ7q1DK6nrvI0b+xVLvkhUnaRxSY/ISIH48csblOVJMvdibNhqWHIkhicG3FwfJV3VTB5o69Cg+1Sh1KP0SFqLCrWcPs7ktRNX7uberx+7v5F+EEExeujB5CRD/9+luqUOwy3ra6MmrxmHbiXrcd9FICSMi6u3KVzajvKEdg+LCpQ7FY6y786bHhqGyqQN7TjWiuqkDKXb6PZH8napvR22LFiEqBcZm9K5zCiaseQki4wf7/pfdeuAlb1Asnrl2AjIC6A2CAkN5Q7vNn+66fnqWN8LxiLNGJGJQXDjmjU3DhvvPw+i0zvq2rSf8q56Huu3s6u8yJj0GYUHet4fJSwCobupwec6hv1wsyS+7deHtkMSIAT3WF0Xl2F/RNNCQiDxmTm6K1CE4NCQxEj8+dAFeu3EKNGoVpnctqd5yjMmLv+qeMgruYl2A00Z+aVVxJWLC1Zg1vHOFwfS/rnV5HylbSBc9ciF0RhOiB9hT5u6PigAAux+5CLER3u1PQ8GjP3sYAcB7t+TjrBGJHo7Ge/KzE/D25hPY6mcrqaibuWYp2OtdAI68+J3Tje24/b0duOGfW3CkulnqcNwSFxGKlGj7c+wFVqsj3DXhsf/hqVUHBxoWEYD+r9SbnZPkV0v6p3WNvJRUNaO+VSdxNNRXHXoj9ld0FpRP5siLb5KXl19+GUOHDkVYWBjy8/OxdetWh+e+/fbbUCgUNl9hYSwuM6uzetEpeHYjnv/ukITRDNyL10/EW4umYfnCqX263ysbjnopIgo27nan9ndJURoMS+5c5WeunSD/sa+iCXqjQFJUKAbHs27Q68nLxx9/jHvvvRePPvoodu7ciQkTJmDu3Lmorq52eJ+YmBicPn3a8nXy5Elvh+k3QlW2/2XPf+d/PSmsRYSqcf6oFFyQm4prpgyWOhwKQn3Z9NP85u+vpnR9Yt9xksmLvzFvxjgxM96vRvy8xevJy7PPPovFixdj0aJFGDNmDF577TVERERg+fLlDu+jUCiQlpZm+UpN7fvUQqBy93d2YmYc5o7t/Hfzl91kjdwDgCRgMLr/ezfcizuw+8LkIZ3JC0de/E8R611seLVgV6fTYceOHVi6dKnlNqVSiYKCAhQWFjq8X0tLC4YMGQKTyYTJkyfjr3/9K8aOHWv3XK1WC622e8OxpqbAXo2id/OFNlKjwpNXjUd20jH8YsogL0flGaYgGb4nedG7sbUGAFw0JhUvXj8Jf/3mAM4fJd9VRs6YayV2lzXCYDRBrWLZo7/Y09VAUYqWF3Lk1d/c2tpaGI3GXiMnqampqKy0v+X8qFGjsHz5cnzxxRd47733YDKZMGvWLJw6Zb9d/LJlyxAbG2v5yszM9Ph1yEWL1uC0uHDGsO7dZU0mID4yFA9dnOvznW77yzovSw/iDcfIt9wdeXljwVSEhajw2OV5OF/GS6SdyUmJQrRGjXa9EQcr/aPgn4D6Vh1Ku7Z2GD8oTtpgZEJ2affMmTOxYMECTJw4Eeeeey5WrFiB5ORkvP7663bPX7p0KRobGy1fZWVlPo7YN77ecxp5j652WuOSatU10x+nYKYP7a6g/+GB8yWMhIJJz5oXjYRtBbxNqVRgYte0A6eO/Mee8s5Rl6GJEWwT0cWrz9KkpCSoVCpUVdnuZFpVVYW0tDS3HiMkJASTJk3CkSNH7B7XaDSIiYmx+QpED/53DwDnu8Jat+L3x2Ht66dn4amrx2PD/50HtUqJp34xXuqQKAj0nIoN5OQF6J462smiXb+x91QDAGD84DhJ45ATrz5LQ0NDMWXKFKxd291EzWQyYe3atZg5c6Zbj2E0GrF3716kp6d7K0y/4E4jLesK9FvPzvZmOF6hVilx7bRMDE3qXNER7O2vyTd+PFJr832ouvv37vHLO2vtCkb734cBR6ZYinYbpA2E3Lab9S69eL3D7r333oubbroJU6dOxfTp0/H888+jtbUVixYtAgAsWLAAgwYNwrJlywAAjz32GGbMmIERI0agoaEBf//733Hy5Enceuut3g5V1tyZBFJarUQKCYBCPKWLlVVCCC4ZpAE5Vd+GR7/cZ3PbpePT8fbmE0iLCcOvZgzB5CHxyPGTujF3TMyKg0IBlNa1oaZZi+RojdQhkQt7OPLSi9eTl1/+8peoqanBI488gsrKSkycOBGrVq2yFPGWlpZCqex+o62vr8fixYtRWVmJ+Ph4TJkyBZs3b8aYMWO8HaqsuVPConL1bu9nXC0+atUZEaXhDhfUf4eqehet/vrcYZg2NAHTsjv7aQTa7r0xYSHISYnCoaoW7Cytx9yx7k3hkzSqmjpQ1aSFUgHkDQrMsoj+8Mkr/5IlS7BkyRK7xzZs2GDz/XPPPYfnnnvOB1H5F+HG2MuQRP9uoNWTq6my+lYdkxcakAOneycv4SEqXDI+sKepJ2fFM3nxE+Yl0jkp0YgI5eudGf8l/IQ7Iy83zx6K043tmNOP/YLkyNU1f7m7AtOzEzBtaILzE4mslNW14bynNzhsOxAMvU8mD4nHR9vKsOtkg9ShkAvdU0aBNQI4UExe/IQ7NS8adWcPikBhcpG9/H11CQDg+LKfsfaF3Pbol/uc9kvquQVHILI0qzvVAJ3BJOmu8+Qci3Xt42+sv/C/ti0D5m7DXXe7DhMBQKvW4PR4iCrwE+FhSZGIDQ+B1mDCgdOB3ZXcnwkhWKzrAJMXmXv9+6P4aneFWzUvgWZeXhrSY8Nw2YQMfHzbDEwban8beIOb7d2J3BEMo3hKpQKT2axO9k7Vt6OhTY8QlQK56YGz4s0TOG0kY/sqGrHs24NOz0mO1qCmWYvf/yzXR1H5TpRGjU0PXgClovMNZcLgOGw70fuFliMv1BdBkJu4ZXJWPNaX1GBnaQMWnSV1NGTP7q5Rl9HpMdCo2ffKGpMXGatp1ro85/c/y8WVkwb7IBppWC//7jAY7Z5zpLoZk7O4TTy5RwH+ngBWO0yz065smVcajRvEepeeOG0kY+6sMAqmF2K9wf4/yNWvFuKtH0/4NhjyW8xxO03IjINSAZQ3tKOqqUPqcMgOc73LBNa79MLkRcZcrbYJNj030LP22Mr9PoyE/NHx2lYcr21l8tIlSqPGqLTOpmccfZEfk0mguLyzmHp8JkdeemLyImPOlnOaBdMLsc5J8kLkTIfeiPOf3oDzn94AnYG/R2YTu94U93btWkzycay2BS1aA8JDVBiRHCV1OLLD5EXG3F0qHCxmDk90eryYL8DkQFOH3vL3Np392ikAiA4LrjLAcYPiADB5kaPdZZ3/J2MzYoKicWJf8V9ExpxNkwSj66Zl4dlrJzg8/rdVzldmUfCyXk2vdDBcmZkQjrX3nuujiOTBXAi651SjWzvXk++YE0r2d7GPyYuMaTm8bUOlVOCqyY5XVoWFcCkh2We0emM+eabV7jlzx6QhJSbMVyHJwsi0KISqlGhs16Osrl3qcMgKtwVwjsmLjHXoHQ9vm3F5cLdwJi/kgNGqF1BTh/0Ou8oA25XdHRq1ytL8bE95g7TBkIXRJLC/q/NxHpdJ28XkRcbanczNmwXfy61j3J+F7BFC4GhNi8vzHE0nBTrz1NHeU6x7kYtjNS3o0JsQEapCdlKk1OHIEl/tZay2xXWTOurGkRey59Xvj2LR29tcnpcWo/FBNPJjnpbYw+RFNoorOv8vxqTH2DTqpG7BVVrvZxx12D13ZDK+P1Tj42jkLyacv87U21OrSpwe/8WUwVArFbghf4iPIpIX84qj4vJGmEwiKKfP5Mbc34VTRo5x5EUmTCaB+ladzW01dkZeZo9Iwjs3T7d8H4wj3f+9YxamDY1HZKjtSAv7d1BfnTsyGX+7ejyevHp80E475qRGQaNWollrwAkHxczkW/squpdJk33B+WyVodvf24FJj6/BXR/uwsxla3Goqtluwe4D80ZJEJ28TBkSj09vn4VZI5Jsbi8qa8CGkmqJoiJ/9OZNU4N+WD5EpcSYrjdJ9nuRnskksI8jLy4xeZGJ/+2vAgB8tbsCpxs78MfPitGh7z2SoFba/peFBfFOoz0/KW87UY+Fb23DvopGt7oTE4Ww+RcAYDyLdmWjrL4NzVoDQtVKjEhhZ11H+MyVqXa90e7IS4iq81Pi7wpGomB0Ks7PTfF1aLIxfWiC3dsveXETLntpk4+jIX/DnXq7mT/h7+HIi+TM9S6j06KZXDvBCkeZMgmBDkPv5MU8xH13QY6vQ5KdG2cMgdEk8OORWqw9aDtdZO6RQORIsG0F4Iy5i+u+8s5Ry2CfSpOSZaVRBpNrZ5jWyVS73mi34yUz8W5KpQI3z852OPpkMgk8/Hkx/rvjlI8jI3/w1yvHSR2CbAxPjkR4iAqtOiOO17ruiUPeY96jLW8Qi3Wd4TuhxP5deAKr91X2uv1YTXfV/1sLp1n+rlbxE1FPjlaJrD1YjX//dBL3fbrbxxGR3F0xMQND2fzLQs2iXVkQQmBfRVexLkdenOK4qYQOVzXj4S/2uTwvUtP938Th3N40DpKXD7eW+jgS8hfB2k3XmbEZMdhxsh4HTjfjyklSRxOcTjd2oK5VB5VSgVFp0VKHI2sceZFQbYvO9UmwHVkIUfK/rKdQB1Np66zqYL7cXeGrcEgm1pdUY9Qfv7V7jHuC9TYmvXPkxdxjhHzPPGWUkxLFjWZd4DuhhNydAoqxKizktFFv7jQX++2Hu3wQCcnJore2OdyZXRPCl76exnZNU+yvaIIQbDUgheIK9ndxF5/BEnKnF8nFeWk2RbqcNuqNRczUV46mGoNZTmoUVEoF6tv0qGzqkDqcoLS/a9Qrj511XeIzWELtdvq49BQbHmKzpLNnkzribtLUdxyS7y0sRIURyZ1N0cwdXsm3uKeR+1iwKyGtG8lLWIgKcRGheP3GKQhVKflGbQf/TainNp3B6fFg7kztzNiMGJRUNWP/6SYUjEmVOpygUtOsRWVTBxQKYHQ6R15c4au+hOy1/+9pXl4aAGDu2LSg7qbrjKOCXQo+RpNAcXkjLn3ReYfloUkRPorIv5iXS++v4MiLr5kLpYclRdqsMCX7+C8kIVfTRuvuOxfDkrm3hSvu1i9sPlLbazNHCix/+Xo/3vrxhMvzLhuf4f1g/JB5xRE7VPueub/LWPZ3cQs/skrI3t5F1jgv7x6Dm5sw3vDmFi9HQlJzJ3G5IDcFSha+22UeeSmta0NTh17iaIILO+v2DZMXCbkaeWEth3uy2SmV+mBIIqeMHImLCMWguHAAwAFOHflUsWWlEUde3MF3Rwm5qnnhck73cISK+uLeC0dKHYKsjebUkc81tukte9lx2sg9fHeUkKtpI468eJ7JzSkmCjxPXjUOh5+4GNFhIVKHImtjWbTrc+Zi3cyEcMRG8PfTHXx3lJDL5IWraNy2fOFUt8674JkNaGhzb1sGCiwFY1LZ0NAN5rqXfUxefMZSrJvOURd38ZksASEEnltzCCv3nLZ7/IqJGbh1djb3X+mDC3JT8cWdZ1m+z0wIt3veiTNteH3jMV+FRTLCxMU95hVHh6uboXOwvQJ5lnmKbiw767qNS6V9TAiBL3dX4IW1h+0ef/baCbhq8mAfRxUYspO7C3ejNCEA2u2e16Z13sCM/NPhqmanxzmS6Z7B8eGICVOjqcOAI9UtlpEY8p4DXckLm9O5j89mH/vTl/tw90dFDo+bm9JR38WEheDj22bg49tmoKnd8TLPd3866cOoyFcufG6j0+Mh3NTULQqFwmrqiDtMe5vWYMSR6hYAwGgmim5j8uJj7xQ6f+Pk3kUDkz8sEfnDEhET7rjoTQigXed6awYKLNzU1H1jumovuOLI+45Ut8BgEogJUyMjNkzqcPwG3yll5L93zOIKIw9xNUOgM3IuP5BsPV7n8hzWkLkvNz0aAFBS6XwqjgbuwOnOf+PR6TH8He0DvlPKyJQh8VKHEDQe+u8enKpvkzoM8pBrXy90eCwiVOX2ajTqlJvWmbwcrGyGEGwv4E2sd+kfJi8+0KYzoPDoGRjZY8RnFHD+Cebb4kosfneHj6IhKT1zzQRckMsdkvsiJyUaSgVQ16pDTYtW6nACmrmfzhgmL33C1UY+cOs727H56Bn830Xs7Okr7oy+HuB8vt+raGh3uZyXnxn6LjxUhaGJkThW24qSymakRLMWwxuEEDhQyZGX/mDy4gObj54BALz3U6nEkQQPzhwHPiEEZj25zuV5Jk579MuotGhL8nJ2TrLU4QSkyqYONLTpoVIqkJMaJXU4foXTRj7k7u7HROSaq41Nzfis65/ctM6RAHNBKXmeefR3WFIk92jrIyYvPtTK5mi+w6r9gNfc4d7ziQWn/TOqq2i3pIrTq95iTgzZCLDvmLz4kLufFGngmLoEvuYOx40IrTF36R/ziqPDVS0wsLWAV+znSqN+Y/JCAcl64OV3BSyUDkTHa91b6s6al/7JSohAeIgKWoMJJ86wrYA3cJl0/zF5oYB3d0GO1CGQh32yvQyL393u1rksNesfpVKBkWlsVuct7TojTtS2AgBGdzUFJPcxefESIQR3ZJUQp40C25+/3Ofw2H0X2o60seal/3JTzc3qWPfiaSVVzTAJICkqlEvR+4HJi5fc+s52TH58DRrbXM/L33n+cKy+5xwfRBU82GY7sDn7/73z/BE23aqZu/SfeZuAgxx58ThOGQ0MkxcvWXuwGi1aA1bvr3R57tWTB1sq+8kzer61vXdLviRxkHc4S02VSgX+c/tMy/cTMuO8Hk+gGsVpI69h8jIwTF68TOvG1BF3kva+2TlJ+Pct03vdfrxrzpkCi0KhwOaHLsBnv5nFDwYDYO71UlrXhha2evAo87YArHfpH75repk7dS/MXTzP3BE03Krx09k5ychKiLA57/ynN+Dp1SX4y8r9Po2PvC8jLhyTsrjZ6UAkRIYiJVoDADhUxdEXTzGZhGUqjiMv/cO3TS9zJ3nhnLzn3X7eMDx51Tisude2luiNBVN6nfuP9Ufw5qbjqGzs8FV4NFAsafIZTh153qn6drRoDQhVKTE8mdsC9AeTFy+zl7ykxYThyyVnSRBN8NCoVbhuehYGx9uOtJiHwe3h6jD/IIRw2F136hCOtHiauVndQW5k6jHm5nQjUqIQouLbcH/wX83LnvvuUK/bOgxGm3l4TQj/G+SAzcz8g6OGaReOScUHi2f4OJrAZ074ueLIc1isO3DcVVoCRpOARq3Cw5eOQYfeyDX+MsGNM/2Doz3CshIiEKrmBwFPM3/QOljZDCEE2xB4gDl54Z5G/cfkRQKmrjfJW2ZnSxwJWeO0kby9uuEoDEYTZgxPtHucb6neMSIlCiqlAo3telQ1aZEWyw9bA3WgkiuNBoofU7zA5OITPD/gy5OOm8/JVpvOgL+tOohn1hxCeX273XM4IOAdYSEqZCdFAmCnXU9o7tCjrK7zd3gMp436jcmLF7iafjCytkJSjoo627QGh1MSJC29sfs5c6ZVZ/ec66dn+SqcoMMVR55jrh1Kjw1DXESoxNH4LyYvXuCq8JN7rUhrxjD70w43vLkFYx9djeYO11s6kG/prUbFHnfQk2cYl5x6zciUzuTlUFWLxJH4PxbregaTFy8wuhp54byRpPQm59NDRWUNvgmE3KZ3MaUXGapyepwGZmRqZ2J4uJojLwPVnbyw3mUgWLDrBa6mjZi7SGuKi66rCpZ+yo7e4PhJ8/ClY1AwOsWH0QSfkV3TRoerWmAyCSiVfI70V/e2ABx5GQifjLy8/PLLGDp0KMLCwpCfn4+tW7c6Pf/TTz9Fbm4uwsLCMG7cOHzzzTe+CNMj3vvpJKb+ZY3UYZATF45JxavzJ9ts3meNr8vyozMaHR67ZXY2hiRG+jCa4DMkIQKhKiXa9UacclAwTa4ZTQIlVdwWwBO8nrx8/PHHuPfee/Hoo49i586dmDBhAubOnYvq6mq752/evBnXX389brnlFuzatQtXXHEFrrjiChQXF3s7VI/44+fFNsWFJD8KhQIXj0vH1KEJiIsIsXuc5EXnZOSFvE+tUmJYcmeCyD2O+u94bSs69CaEhSgxlAn3gHg9eXn22WexePFiLFq0CGPGjMFrr72GiIgILF++3O75L7zwAubNm4f7778fo0ePxuOPP47JkyfjH//4h7dD7ZfmDj0+2V6GxjbXRZ435Heuhnj5hsneDovcZC5EtMbcRXpHqptx7euF2HykFgBQ32Z/hRH5jmXFEZOXfjPXu4xKi4GKQ7wD4tXkRafTYceOHSgoKOj+gUolCgoKUFhYaPc+hYWFNucDwNy5cx2er9Vq0dTUZPPlS/d+shsP/GcP7vpol8tzb5iehYOPz8Ml49N9EBm5Y25eWq/bXPXpIe+77d87sPV4HW54c0vn9+9ulzgiGplqrnth8tJfls66LNYdMK8mL7W1tTAajUhNTbW5PTU1FZWVlXbvU1lZ2afzly1bhtjYWMtXZmamZ4J305r9VQCAjYdqXJ6rUHQ2fCL5WDhraK/b2KxOetVNWpvvW3WOa17IN3JSOlcccbl0/3GZtOf4/VLppUuXorGx0fJVVlYmWSxHa5w/qTlMKD8qpQJpMbbtzpf/eALVTR0SRUSA+63+B8eHezUO6maeNjpS08J2D/104HTnqBU76w6cV5OXpKQkqFQqVFVV2dxeVVWFtLTew/UAkJaW1qfzNRoNYmJibL6kMueZ750eN7CQV5Z6JpUbD9Xgujd+kigaAgB3nylnDU/yahzULTM+AmEhSugMJpw80yp1OH6nvlWHyq4PRblMXgbMq8lLaGgopkyZgrVr11puM5lMWLt2LWbOtL9MdebMmTbnA8CaNWscnu9POvQc+pajEFXvz/nHavni7A+E22kODZRSqcAITh31m3nKKCshAlEatlgbKK9PG91777345z//iXfeeQcHDhzAHXfcgdbWVixatAgAsGDBAixdutRy/t13341Vq1bhmWeewcGDB/GnP/0J27dvx5IlS7wdqte5al5H0mDDLXk74qSrK59SvsWi3f7bz866HuX19O+Xv/wlampq8Mgjj6CyshITJ07EqlWrLEW5paWlUCq7c6hZs2bhgw8+wB//+Ef8/ve/R05ODj7//HPk5eV5O1SvOjsnCdOGJkgdBtnDN0BZK3h2o8NjN5+V7cNIyJy8cLl035nrXVis6xk+GbtasmSJw5GTDRs29LrtmmuuwTXXXOPlqHznzQVTUTAm1fWJJAlXG2mSPKVEazAmg28EvmTZ44jTRn22nyuNPMrvVxvJ2SOXjsF3957LxEXm7C2XJvm7if9vPmceeTlW2+Jys0zqpjOYLNOfXGnkGUxevOjS8emWAjeSrwUzh0odAvXROzdPx6/PGSZ1GEFnUFw4IkNV0BsFVxz1wdGaFuiNAtEaNZf3ewiTFy9Sq/jP6w8cFezWNGvt3k7e16I1OD1+7shkPr8koFAoMMJc91LJqSN3mVca5aZHc+80D+Gz34vsLcEl/1Fa1/nJUmfg8LgvNbjYx+i1X3FvMCmNSjUvl2bRrrvYWdfzmLx4UQg/Gfq1Dr0J/y48gZF//Bbfu7H9A3nG/grH+5O9f2s+5uVxbzApWZZLO1nCTra40sjz+O7qRUxe/Meyq8b1uq1dZ8TDX+wDAPz2Q9cbb5JnmDdjtIdbbEgvxzJtxOTFHUIIjrx4Ad9dvWRiZhxfaP3I9dOz8P3959nc1mHo7ogsuJxaFtR8TkluVFfycuJMG7QGdg13paZZizOtOigV3f92NHBMXvqpXWfE/3262+6xzIRwfPabWT6OiAYqMUpj8307dzL2OVdJIot0pZcao0F0mBpGk8BxbqPhkrm/S3ZSJMJDVRJHEzj4StBPy388jv/sOGX3WESImhXlfiiyxwtLBwt1fc7VFhoceZGeQqHo7rTLqSOXWO/iHUxe+qmqa3dQe0LUfIH1Rz0Tzg6rkRdOGvmGq8ZnnIqVh+49jrhc2hXWu3gHk5d+EELg3cKTDo+Hh3BoMBCYh3vJd1wtS9eo+ZIlByO5XNpt5tcRdtb1LL4S9MN3B6qdHg9j8uK33lo0zfL3z3aVdx8QQGObHkZuY+xVOhcjLzHhIT6KhJzpXi7NkRdnOvRGHKvp/DfiyItnMXnpB1dtsZm8+K/zR6XgjvOG97q9WWvAhMf+h+v/+ZMEUQWHI9XNeGX9UafnRIf5ZC9ZcmGkZcVRKzr0LGx35FBVM0wCiI8IQWqMxvUdyG18JeiHVq3zJyuTF//mbNpv6/E6H0YSXAqe3ejyHI2azy05SIoKRXxECOrb9DhS3YK8QbFShyRL1vUuXMThWRx56YdWnfN9Vzgv79/CQvj/52tHazj94E8UCoWlWR077TrGlUbew1fpfmh1sWmcgVvF+zUWXPveL18vdHnOL6YM9kEk5K5R3KDRpf1caeQ1nDbqB1dFm1r2B/FrGiYvPlfb4nwzxg9uzcesEUk+iobcYV5xdJgrjuyy3RaAnXU9jSMvAxRqp+Pn4PhwCSIhT+HIi/yEcipWdsxFu4c4bWRXeUM7mjsMUCsVGJESJXU4AYevCAM0LDnS8vdHLxuDuWNTcXfBSAkjooFylbxwubTvcVsA+TEnL2V17S6n0oORud5lREoUC829gNNGA3TR2DT8YspgjB8ch+nZCVh0VrbUIdEAuVotpjOYuEeJj3GJtPzER4YiKUqD2hYtjlS3YEJmnNQhycoBNqfzKn6cGSC1UoFbzx6G6dkJUodCHhIe6vxp8Zev9/sokuDhrOv/HecNx/BkDrvL0ag0dtp1hNsCeBeTlz76fFc5PtpWZvmee60EHldDvO9vKfVRJMGhuqkDzmbiHpyX67tgqE9yUrrqXpi89MKVRt7F5KUPjCaB+/+z2+Y2JRsPBRzrKaFfnzPM7jlCCNS3Ol8hQ+6Z/dR6qUOgfhqVZk5euFzaWovWgJNn2gBwpZG3MHnpg6qmDuiNth8RWUcYeKxrXmIj7O+l8/vP9mLS42vww+EaX4UVsFxtxkjyxQ0a7Sup7Bx1SYnWIDGK2wJ4A996++BUfXuv2zjyEnisVxvFhYfaPefDrZ1Th8+tOeSTmIjkaETXtNHpxg40degljkY+9netNBqTwSkjb2Hy0gen6tt63caal8BjvT1AcjQ/NUlp3tg0qUMgJ2LDQ5AeGwaAzeqs7a9gvYu3MXnpA3sjL0xeAk+YVcFuQqT9kRczdnzpP4PRhGf/V+Lw+KC4cLz6q8k+jIj6w7zHEeteunGlkfcxeemDOjsFmtwpNPAolQpcMTEDs4YnYmJmHK5xsqfOvoomrD9Y7cPoAscHW0vx4rojvW4PUXU+p66aPIjPLz8winUvNowmgZLKrmkjJi9ew85PfaC3s+Giii+uAen56yZZ/v6XK/Pw6Y5Tds/TGUxY9PY2/PeOmZgyhL1++sL86bSnj26biZrmDlyQm+rjiKg/ukdemLwAwIkzrWjXGxEWokR2UqTrO1C/cOSlDwzG3pME5k+JFLjcae399Z5KH0QSWNp1Rru3J0WFYl5eOvcz8hMjOW1kw5yUj0qNZlmBF/HVoQ/0pt4jL5eOz5AgEpKbXWX1+KKoHEKwCsZdHXr7S6S5gs+/5HRtOljTrGXvI1htC8CVRl7F5KUPeo68vHDdRO5xQwCAXaUNuPujInx/iH1f3NWutz/youZopl+J1KgxOD4cAKeOAK408hUmL31g6DHyEsIOddTDvgr7dRzUW4eD5EUBJi/+ZhTrXizMu0mzWNe7+O7bB7276/JFlmxxysN9BgcbGpk49eZ3uFy6U12rDpVNHQCAXCYvXsXkpQ8Mxp4jL3yjIlvMXdznKElh6uJ/uLt0J3O9S1ZCBKI0XMzrTUxe+qDnJ8W4COcNzChw/PDA+Xj35ukcbfOQVcWnsau0we6xtJgw3wZDA2a9u3QwF61binU56uJ1TF7cJITAmRbbSvp4Ji9BIzMhAueMTIbRwVSH2SfbyvDZLvs9Yajb7e/ttHv7h4tnMEH0QyNSoqBUAPVtetS2BO+Ko/3srOszTF7ctP90k+UX0yzewY7DFLyO1bbidx/v7jXFSN3sNXs047SbfwoLUWFIYmdDtmCeOupeaRQtcSSBj8mLm+yNssSEMXkh+3RMXhxq1RocHrPXCJL8g7nfS7AmLzqDCUdrOguW2ePF+5i8uMk6ebl/7ij88MD5UHJ4mxzQOmjARo5XGQHOR2VI3kalBfdy6SPVLdAbBWLC1BgUFy51OAGPyYubrJvRJUdpkJkQIWE0JHcceXGsplnr8JizxIbkLdiXS5vLCnLTY7ihqA8weekHe9sEUPAp+cs8LF841e4xjrzYJ4TAxS/84PA4a4X810ir3aWDccURVxr5FpOXPrjrghEYkx6DyycOkjoUksid5w9HTJgaG+8/Hxq1CvnZiXbP0xntd48Ndvd9utvp8agw9sbwV8OSoqBWKtDcYbA0agsmTF58i68UfXDfRaNw30WjpA6DJHT/3Fzce+Eoy3Le8BD7e1s52nQw2K3YWW739mumDEakRo3ZI5J8HBF5SqhaiaFJkThS3YJDVS1Ijw2eug8hBJdJ+xhHXoj6yLoPiaOibda89M1107Pwp5+PZa2An7PscVQZXEW7lU0daGjTQ6VUIKdr+oy8i8kLkRew5qW3wqNnHB6L5nRRQMhJDc7l0uYpo+HJkQhzMBpLnsXkhcgLOPJiy2A04fp//mT32HXTMjEylU29AkGw7i5tbk7HehffYfJC5AVaPQt2rbXqHP97PHn1eB9GQt5kXi59uLoFpiBa9n7gdGeyxnoX32HyQuQFHHmxteBfW6QOgXxgaGIEQlVKtOmMKG9olzocnznAYl2fY/JC5AWseen045FaLH53O3afapQ6FPIBtUqJYcnBtcdRm86A42daATB58SUmL0QD9JSdaY/TjcHzqdOZ+W9uwZr9VVKHQT40Msg67R6sbIYQQHK0BsnRGqnDCRpMXogGKCGy96adT//vEL4/VAO90QQhBBradBJEJq1Nh2ulDoEkEGx7HHXvJM1RF1/i+kSiATpnZLLd229avhVzx6YiVK3CV7sr8N87ZmHKkHgfRyeN59YcwgtrD0sdBkkg2HaX3lfROSWax52kfYojL0QDFKp2/DRava8KX+2uAAC8sfGor0KSHBOX4GWeNjpS3QJjEKw42tc18pI3KFbiSIILkxciH1E56MYbTH57wQib76cNDY6RqGCSmRCBsBAltAYTSuvapA7Hq/RGEw52LZMey5EXn2LyQuQjSra+x70XjcKPD12AYUmRmJObgn8tnCZ1SORhKqUCI4Jk6uhIdQt0RhOiw9TISoiQOpygwuSFyEdW7jkdsI27jta04LGv9qPayW7C0ZrOErtBceFY93/n4V8LpyEmLMRXIZIPjQySPY6KyzvrXcakx3BfLh9j8kLkQyv3npY6BK+46pXNWP7jcdz14S6H53yweIYPIyIpWZKX6sBeLs16F+kweSHygCeuzHPrvPL6wOz/0tiuBwDsLK13OLoUHsoN64KFeY+jksomiSPxLvNKI9a7+B6TFyIPmJ8/BN/89mzEuNgd2SQCc9rIzCSADoP9fYxCVBxWDxbmXi/HalqhdfD74O9MJmHp8cKRF99j8kLkIWMyYnDXBTlShyEpo0mgwsGeNiEqvtwEi/TYMMSEqWEwCRytbpU6HK84caYVrTojNGolhiVFSh1O0OGrCZEHqV2MLgRqwa61gmc32r3d1b8NBQ6FQmHpOGvetDDQFFt11lUzMfc5/osTeZCr0YXAT10cC1Hy5SaYmJOXgwFa92LprDuI9S5S4KsJkQdZ13X8/me5vY4HeMmLUyFOOhFT4Mntqns5GKDLpfeVdyZlYzNY7yIFvpoQedD4wXGWv88bm97reKAX7DqjZofhoBLI00ZCCKs9jZi8SIEbMxJ50Oj0GHxwaz7SYsOgslPjEbypCwt2g83I1GgoFEBtiw41zVokR2ukDsljKho7UN+mh1qpwMi0KKnDCUpefTWpq6vD/PnzERMTg7i4ONxyyy1oaXHetOi8886DQqGw+br99tu9GSaRR80akYRhyVH2RxqCeOSFezsFl/BQFbITO1fhBFrdy76uzro5qdHQqNm/SApeTV7mz5+Pffv2Yc2aNVi5ciU2btyI2267zeX9Fi9ejNOnT1u+nnrqKW+GSeQV9t6sgzV1WXnXbKlDIAnkpnfWvQTa1JF5pRGb00nHa8nLgQMHsGrVKrz55pvIz8/H7Nmz8dJLL+Gjjz5CRUWF0/tGREQgLS3N8hUTw18Q8j8qO3udtOuMOHmms++FEALvbzmJXaX1vg7N5yI1nKEORqPTulYcnQ6sol3zyEsekxfJeC15KSwsRFxcHKZOnWq5raCgAEqlElu2bHF63/fffx9JSUnIy8vD0qVL0dbmeFt1rVaLpqYmmy8iObBX8/LmpuM49+8bsLusAetLqvGHz4px5SubJYhuYITV9NdLaw+7PJ/FusEp11y0G2ArjrinkfS89nGosrISKSkptj9MrUZCQgIqKysd3u+GG27AkCFDkJGRgT179uDBBx9ESUkJVqxYYff8ZcuW4c9//rNHYyfyBGdv2J8XlSMlOsyH0XjOidpWXPXqZiyaNRR3zcnBM2sOubyPkslLUDIvlz5S3Qy90RQQRdu1LVpUNnVAoeheUUW+1+ffpIceeqhXQW3Pr4MHD/Y7oNtuuw1z587FuHHjMH/+fLz77rv47LPPcPToUbvnL126FI2NjZavsrKyfv9sIk9yVqAaFx7q9uMImRX5PvfdIdS16txKWs4flYxpQ+ORFuOfiRoNzOD4cERr1NAbBY7WBMYO03u7poyykyI5HSqhPv/L33fffVi4cKHTc4YNG4a0tDRUV1fb3G4wGFBXV4e0tDS3f15+fj4A4MiRIxg+fHiv4xqNBhpN4CzBo8ChdtJR1mgy4W+rut/8hRBQ2KmRueejXThwuhlf3TUboTJp8ubsuqwNjg/HW4umezkakjOFQoHc9GhsO1GPg6ebkZvm/yMVe8o6k5cJVj2dyPf6nLwkJycjOTnZ5XkzZ85EQ0MDduzYgSlTpgAA1q1bB5PJZElI3FFUVAQASE/v3fCLSM6czZS8uO6IzfcdehPCQ3svufy8qLO4ffPRWpw3KqXXcSmEhbiXvHy5hCuMCMhNi8G2E/U4UNmEKzBI6nAGbG95AwBgHOtdJOW1j3KjR4/GvHnzsHjxYmzduhU//vgjlixZguuuuw4ZGRkAgPLycuTm5mLr1q0AgKNHj+Lxxx/Hjh07cOLECXz55ZdYsGABzjnnHIwfP95boRJ5hb2RFEdadQanx+U0cRQW4l5fi4RI96fGKHB1L5f2/6JdIQR2n+oaeclk8iIlr45Dv//++8jNzcWcOXPws5/9DLNnz8Ybb7xhOa7X61FSUmJZTRQaGorvvvsOF110EXJzc3Hffffh6quvxldffeXNMIm8Ljsp0unxNq3RR5EMXLhV8vLt3tN2z/nTZWN8FQ7JnGWDxgDo9VLZ1IGaZi1USgXGpDN5kZJXq40SEhLwwQcfODw+dOhQm2LEzMxMfP/9994MiUgSrsZg7I28GE3dzw05rdWxnja64/2dvY6PTI3CwrOyfRkSydio1M6Rl+pmLc60aJEY5b81inu6Rl1yUqLsTvOS78ijApAowKW6WG1TYqcPht5o8lY4A+Jq2XNfVlJR4IvUqDEkMQKA/+8wvedUAwAW68oBkxciL8rPTkCURo2X5092et49HxdBa7CdOjJYj7z0oX7G2xQuxoFkFCrJhLnT7v4K/546Mo+8jGe9i+SYvBB50YeLZ2DHwwVIiAzFU1ePR3qs4xGYxja9zfcGq5EXueQD7TojSlxsssfkhXrKG9SZvOyraJQ4kv4TQnQnL4PipA2GmLwQeZNSqbDsOnvttEwULp1jczzJav6/qcM2edEbu0deTD5qVGc0Cfzpy31Yuaf3/mMdeiNGP7LKsnybyF1ju5YVF/vxyEtpXRsa2/UIVSkxqqtzMEmHyQuRhKynihrbe4y8mLpHXqyLd71FCIFn15Tg7c0nsOSDXb2OH65yr0Oqq2klCj55GZ3Jy9GaFrS5aAsgV+Yl0qPTo2XTMDKY8X+ASEL52YmWvze263Hn+zvx+8/2AgAMViMv1qMw3vLZrnK8vN7+NhwAcKjKvWLL+y4a6amQKEAkR2uQGqOBEMABP10yvaesAQAwnsW6ssDkhUhCT1/T3XyxqKwRX+89jQ+2lEJnMNmsNvLFyMvyH487PKY1GHHfp7tdPsa+P8/F1KEJngyLAoR59KW43E+Tl649jcYPZrGuHDB5IZJQXEQoLs7r3OuroU1nub1Va7BZbWQ9heQtzvYsOlbT6tZjcKM6csRS91Luf0W7RpOwxM2RF3ngKw2RxGLDQwAAtS1ay22fF5Wjurn7e19MG6md9G85Vd/u9Z9PgS0vo3PFkT8W7XbW6hgREarCiJQoqcMhMHkhkpw5ealq6k5W/vzVfptzjD4YeXHWfO7kGfdGXogcyesaeTlc1YwOvdHtPbLkwLxEOi8jFioXTRrJNzhtRCSxmK7kZcfJeofnSDHyYq6z+XBrKf7y9QGv/3wKbOmxYUiIDIXBJNwu/paLXaWdz03Wu8gHkxciiUW4sUeKdcO68oZ2vLLhSK+mdgOlVtm+HPz63zsAAA9/XuzRn0PBSaFQYKx56sjPinZ3lTYAACYPiZc2ELJg8kIkMXeGz61HXq59rRBPrSrBQyv2eDSOniMv3x2oAmC7TYEzq+85x6PxUODJszSr85+i3VatAQe7ukpPzmLyIhdMXogkZr1LsyPW3XfLGzqLZzeU1Hg0joHO5bPrKLliXi69z49WHO051QiT6Jz2SnOyvQf5FpMXIomFqV2PvLy07giWb7Ltw+LJLQMa2/UQdh7v3o+LPPYziMx7HB2obJbtruk97eyqd+Goi7wweSGSmLurLh5babsCSWsw2U04+qq6uQMT/vw/fHegutexFbvK3XoMZxtOEpllJUQgOkwNncGEI9XubTchNXOx7qSsOGkDIRtMXogkpnFj2siRx1cOfBXQ+oO9kxZ3adRKvH7jFHy5ZPaA46DAZ120u/eU/KeOhBCWYt1JHHmRFSYvRBIx15horKaNrp482Ol9fjxSa/N9z5b+eqMJd324C/8uPOF2HFpD34fvLxmXDgBYMHMI5o5NQ3K0xsU9iDpN6OpQW3SqQdI43FFa14YzrTqEqpSWKS+SByYvRD728wkZAIA7zh0OAGjXde8sveyqcU7v66o/xrfFlfhqdwUe/mKf2/Ho+pG8vHDdRCxfOBV3zcnp830puE3MjAMAFHWNaMiZedRlTEaMzYcMkh477BL52N+vGY+bZg2xfAI1f6JLjdEgVO3884RS0XtFkN5oQkhXj5Z2ncHtOJo79Fh7oBpnWnWuT+5BrVLigtzUPt+PaGJX7UhJVTPadUaEu9HnSCos1pUvJi9EPqZRqzBlSPfOy3ERodj58IWWZnVTh8Rju4Nuu/ZWM+f84Vvcef5w3D8312HyI4SAokfis3TFXqzcc7pPsafHhuGZayb06T5E1tJjw5Eao0FVkxbFFY2YJuNdyHeyWFe2OG1EJAMJkaGWVUdv3jTV4XlfFFXYvf3l9UcBAKGq7k+x5umgdp0RFzzzPe77ZDc2Ha61rJ7oa+ICAJ/ePhOzRiT1+X5E1ix1LzKeOmrXGXHgdOc0LTvryg+TFyKZiYsIxcOXjrF7zNGIjFmIqnt0xVxLc+u723C8thX/3XkKv/rXFlz5ymYA9kdxXEmMZGEuDZx56qiorEHSOJzZVVYPo0kgLSYMGWwFIDtMXohk6KaZQ/p1P+upoU93lGF9STV+PHKm13k6g8mym3VfyLk+gfyHpWhXxsnLtuOdHxSmZSf0mnIl6bHmhUiGem6S6I51B6tQbNV23dlO0JuO1CA3LQaFx3onNo7kZ8u3NoH8y7hBsVAoOre6qGnWynKp/bYTdQCA6UM5ZSRHTF6IAsTNb2/36LlXTx6M/+48BQBYenEuFp89rN+xEVmLDgtBTkoUDlW1oKisAReOkdfKNb3RhB0nu0deSH44bUREdoWHdr88xEeEQjnAjRuJrHVPHTmv45LCvoomtOuNiA0PwcgUbjgqR0xeiMiucKs9l8JY60IeNqEredldJr9tArYd75wymjY0nkm7TDF5ISK7bJIXF83ziPpqoiV5aYDJ5Lkd0j1h6wlz8sIpI7niKxIR2WU92uLuztdE7hqVGo3IUBWatQaUuNj2wpdMJoHt5uSF9S6yxeSFiOyyHnlRceicPEytUlqav5lX9sjBkZoW1LfpER6iQl5GrNThkANMXojILuvRFra5IG8wT8tsOyGfot2tXfUuk7LiXO41RtLh/wyRH4iQoGBWY/XCbW9DSKKBmtrVQ2Xb8ToIIY+6l8Kjnb2P8rMTJY6EnGHyQuQH2rpa/fuS9VRRRmy4z38+Bb5JmfEIUSlQ2dSBU/XtUocDk0lg89FaAMBZI5i8yBmb1BGRXQqFAp/8eibqWnXISoyQOhwKQOGhKuQNisWu0gZsPV6HzARpf88OVDahvk2PyFCVZSk3yRNHXoj8QGqM99unJ0VpMDi+e4RFqQCmZydgXl6a1382Ba/plroX6Yt2fzzSOeoyPTsBIf3YooN8h/87RH7g3ZvzceWkQcjqwyfTaE3fBlZ/fOh8fHfvuZbvWedCvjBNVslLZ73LWSOSJI6EXGHyQuQHRqVF47lfTsQHi/Nx3bRM/PnnY13eJzlagz9dNgZhIbZP83NHJtt8v+I3s7DyrtnQqFU2K4y4Opp8YUrXcumjNa0406KVLA6dwWRZacTkRf6YvBD5kcHxEXjy6vGYneP4xfXft0zHpKw4vPqrKVh4VjaK/zTX5njPHXwnZ8Ujb1B3P4vQruHySVncTZe8Lz4yFCNTowBIO/pSVNaAdr0RiZGhGJXK/YzkjskLkR8anhyF53850e6xs3OS8dlvzsKotM4XYLXV3H2oWmnTiv2G/Kxe99/5yIX4aekcpMaEeTZoIgdmDOtc2bO5a5myFMz1LjOHJ3I/Iz/A5IXIT10xaZDb575+4xQkR2vwzqLpMFr10/jrleN6nRulUSMtlokL+Y55mmZTVwIhBXPywikj/8Cl0kRBYO7YNFw0JhUKhQKHq5vxRVGF1CERWcwcngilAjhW04qKhnZkxPm2r1Bjmx67yhoAALOZvPgFJi9EQULRtXrohulZUCuVyB/GTedIHmLCQjAhMw67Shuw6Ugtrp2a6dOfv/FwDYwmgZyUKMl7zZB7OG1EJFMXjUkF0Ht1kLXV95yDW2dnY8vv52B+fhaWL5zq8nHVKiVuyM/C8OQoj8VKNFDmEY8fJZg6Wn+wGgBwfm6Kz3829Q9HXohk6plrJ2DN/ioUdCUx9oxKi8YfLx0DAHjCTv0Kkb84a0QSXlp3BD8eqYUQwjJS6G0mk8CGQzUAgPNHMXnxFxx5IZKp6LAQXDV5MGLCQqQOhcjrJmXFITxEhdoWHQ5WNvvs5+4+1YC6Vh2iNWrLRpEkf0xeiIhIchq1CtOzO+uwfjhc47Ofa54yOntkErcE8CP8nyIiIlk4b1RnfdfaA9U++5nrSzhl5I+YvBARkSwUjO6s79p+sh6NbXqv/7zyhnbsLW+EQgGcx+TFrzB5ISIiWchMiMDI1CgYTQIbDnl/9GVVcSUAYNqQhF7bZpC8MXkhIiLZmNM1+vKdD6aOvt17GgBw8bg0r/8s8iwmL0REJBsFozunb74vqYbeaPLaz6lq6sCO0noAwLw8Ji/+hskLERHJxsTMeCREhqKpw4Atx7y3y/TqfZUQonOJdnqsb7cjoIFj8kJERLKhUiowd2zn1NHKPd7bg2vlnq4pI466+CUmL0REJCuXjc8AAKzaVwmdwfNTR2V1bdh6vA4KBXDZhAyPPz55H5MXIiKSlfxhiUiK0qChTe+VvY4+21UOADhreBKnjPwUkxciIpIVlVKBS7pWAH3l4akjIQRW7DwFALhq8iCPPjb5DpMXIiKSnUu7pnP+t68KbTqDxx53Z2k9TpxpQ0SoiquM/BiTFyIikp0pWfHISohAi9aAr7uKaz3hw61lAICL89IREar22OOSbzF5ISIi2VEqFfjltEwAwEfbyjzymHWtOny5u3Ma6lczsjzymCQNJi9ERCRL10wZDJVSgR0n63GoqnnAj/fxtjLoDCaMGxSLiZlxAw+QJMPkhYiIZCklJgxzcjs77r6z+cSAHktvNOG9n04CABbMHAKFQjHQ8EhCTF6IiEi2Fp2VDQD4z45TqG3R9vtxPt9VjvKGdiRFhbK3SwBg8kJERLI1Y1gCJgyOhdZgwrv9HH0xmgRe2XAUALD47GEIC1F5MEKSApMXIiKSLYVCgV+fOxwA8PbmE2ho0/X5Mb4oKsfx2lbERYTgVzOGeDpEkoDXkpcnnngCs2bNQkREBOLi4ty6jxACjzzyCNLT0xEeHo6CggIcPnzYWyESEZEfmDs2DaNSo9HUYcA/1h3p031btQb8bdVBAMBt5wxDpIbLowOB15IXnU6Ha665BnfccYfb93nqqafw4osv4rXXXsOWLVsQGRmJuXPnoqOjw1thEhGRzKmUCiz9WS4A4J3CEzh5ptXt+76y4QiqmrTISojAzV31M+T/vJa8/PnPf8bvfvc7jBs3zq3zhRB4/vnn8cc//hGXX345xo8fj3fffRcVFRX4/PPPvRUmERH5gfNGpeDsnCTojQIP/ncPTCbh8j57TzXijY3HAAB/uGQ0a10CiGxqXo4fP47KykoUFBRYbouNjUV+fj4KCwsd3k+r1aKpqcnmi4iIAs/jl+chPESFn47V4Y0fjjk9t6lDj7s/3gW9UWDe2DRcNCbVR1GSL8gmeamsrAQApKba/oKlpqZajtmzbNkyxMbGWr4yMzO9GicREUljaFIk/njpaADA31YdxDd77W8b0KE34tfv7sCxmlakxYRh2VXj2NclwPQpeXnooYegUCicfh08eNBbsdq1dOlSNDY2Wr7KyjzTRpqIiOTnhulZ+NWMLAgBLPlgJ9784RgMRpPleFldG375xk8oPHYGURo1/rVwKuIjQyWMmLyhT2XX9913HxYuXOj0nGHDhvUrkLS0zt09q6qqkJ6ebrm9qqoKEydOdHg/jUYDjUbTr59JRET+RaFQ4E+XjYVWb8KnO07hL18fwHs/ncSMYYmobdFi46Fa6IwmxEWE4J8LpmJsRqzUIZMX9Cl5SU5ORnJyslcCyc7ORlpaGtauXWtJVpqamrBly5Y+rVgiIqLAplYp8dQvxmN8Zhye+V8JTpxpw4kzbZbjs4YnYtlV4zAkMVLCKMmbvLbgvbS0FHV1dSgtLYXRaERRUREAYMSIEYiKigIA5ObmYtmyZbjyyiuhUChwzz334C9/+QtycnKQnZ2Nhx9+GBkZGbjiiiu8FSYREfkhhUKBG2cMwRUTM7ChpAbHaloRqVFhxrBEjM2IYY1LgPNa8vLII4/gnXfesXw/adIkAMD69etx3nnnAQBKSkrQ2NhoOeeBBx5Aa2srbrvtNjQ0NGD27NlYtWoVwsLCvBUmERH5seiwEO5VFIQUQgjXi+X9SFNTE2JjY9HY2IiYmBipwyEiIiI39OX9WzZLpYmIiIjcweSFiIiI/AqTFyIiIvIrTF6IiIjIrzB5ISIiIr/C5IWIiIj8CpMXIiIi8itMXoiIiMivMHkhIiIiv8LkhYiIiPwKkxciIiLyK0xeiIiIyK94bVdpqZj3mWxqapI4EiIiInKX+X3bnf2iAy55aW5uBgBkZmZKHAkRERH1VXNzM2JjY52eoxDupDh+xGQyoaKiAtHR0VAoFB597KamJmRmZqKsrMzldt3+jtcauILpenmtgYnXGpiEEGhubkZGRgaUSudVLQE38qJUKjF48GCv/oyYmJiA/yUy47UGrmC6Xl5rYOK1Bh5XIy5mLNglIiIiv8LkhYiIiPwKk5c+0Gg0ePTRR6HRaKQOxet4rYErmK6X1xqYeK0UcAW7REREFNg48kJERER+hckLERER+RUmL0RERORXmLwQERGRX2Hy4qaXX34ZQ4cORVhYGPLz87F161apQ+qzZcuWYdq0aYiOjkZKSgquuOIKlJSU2JzT0dGBO++8E4mJiYiKisLVV1+Nqqoqm3NKS0txySWXICIiAikpKbj//vthMBh8eSl99uSTT0KhUOCee+6x3BZI11peXo5f/epXSExMRHh4OMaNG4ft27dbjgsh8MgjjyA9PR3h4eEoKCjA4cOHbR6jrq4O8+fPR0xMDOLi4nDLLbegpaXF15fiktFoxMMPP4zs7GyEh4dj+PDhePzxx232Q/HX6924cSMuu+wyZGRkQKFQ4PPPP7c57qnr2rNnD84++2yEhYUhMzMTTz31lLcvrRdn16rX6/Hggw9i3LhxiIyMREZGBhYsWICKigqbxwiEa+3p9ttvh0KhwPPPP29zu79cq88Icumjjz4SoaGhYvny5WLfvn1i8eLFIi4uTlRVVUkdWp/MnTtXvPXWW6K4uFgUFRWJn/3sZyIrK0u0tLRYzrn99ttFZmamWLt2rdi+fbuYMWOGmDVrluW4wWAQeXl5oqCgQOzatUt88803IikpSSxdulSKS3LL1q1bxdChQ8X48ePF3Xffbbk9UK61rq5ODBkyRCxcuFBs2bJFHDt2TKxevVocOXLEcs6TTz4pYmNjxeeffy52794tfv7zn4vs7GzR3t5uOWfevHliwoQJ4qeffhI//PCDGDFihLj++uuluCSnnnjiCZGYmChWrlwpjh8/Lj799FMRFRUlXnjhBcs5/nq933zzjfjDH/4gVqxYIQCIzz77zOa4J66rsbFRpKamivnz54vi4mLx4YcfivDwcPH666/76jKFEM6vtaGhQRQUFIiPP/5YHDx4UBQWForp06eLKVOm2DxGIFyrtRUrVogJEyaIjIwM8dxzz9kc85dr9RUmL26YPn26uPPOOy3fG41GkZGRIZYtWyZhVANXXV0tAIjvv/9eCNH5ghESEiI+/fRTyzkHDhwQAERhYaEQovNJqFQqRWVlpeWcV199VcTExAitVuvbC3BDc3OzyMnJEWvWrBHnnnuuJXkJpGt98MEHxezZsx0eN5lMIi0tTfz973+33NbQ0CA0Go348MMPhRBC7N+/XwAQ27Zts5zz7bffCoVCIcrLy70XfD9ccskl4uabb7a57aqrrhLz588XQgTO9fZ8k/PUdb3yyisiPj7e5nf4wQcfFKNGjfLyFTnm7A3dbOvWrQKAOHnypBAi8K711KlTYtCgQaK4uFgMGTLEJnnx12v1Jk4buaDT6bBjxw4UFBRYblMqlSgoKEBhYaGEkQ1cY2MjACAhIQEAsGPHDuj1eptrzc3NRVZWluVaCwsLMW7cOKSmplrOmTt3LpqamrBv3z4fRu+eO++8E5dcconNNQGBda1ffvklpk6dimuuuQYpKSmYNGkS/vnPf1qOHz9+HJWVlTbXGhsbi/z8fJtrjYuLw9SpUy3nFBQUQKlUYsuWLb67GDfMmjULa9euxaFDhwAAu3fvxqZNm3DxxRcDCLzrNfPUdRUWFuKcc85BaGio5Zy5c+eipKQE9fX1PrqavmtsbIRCoUBcXByAwLpWk8mEG2+8Effffz/Gjh3b63ggXaunMHlxoba2Fkaj0eYNDABSU1NRWVkpUVQDZzKZcM899+Css85CXl4eAKCyshKhoaGWFwcz62utrKy0+29hPiYnH330EXbu3Illy5b1OhZI13rs2DG8+uqryMnJwerVq3HHHXfgt7/9Ld555x0A3bE6+x2urKxESkqKzXG1Wo2EhARZXSsAPPTQQ7juuuuQm5uLkJAQTJo0Cffccw/mz58PIPCu18xT1+Uvv9fWOjo68OCDD+L666+3bE4YSNf6t7/9DWq1Gr/97W/tHg+ka/WUgNtVmtxz5513ori4GJs2bZI6FK8oKyvD3XffjTVr1iAsLEzqcLzKZDJh6tSp+Otf/woAmDRpEoqLi/Haa6/hpptukjg6z/vkk0/w/vvv44MPPsDYsWNRVFSEe+65BxkZGQF5vcFOr9fj2muvhRACr776qtTheNyOHTvwwgsvYOfOnVAoFFKH4zc48uJCUlISVCpVr1UoVVVVSEtLkyiqgVmyZAlWrlyJ9evXY/DgwZbb09LSoNPp0NDQYHO+9bWmpaXZ/bcwH5OLHTt2oLq6GpMnT4ZarYZarcb333+PF198EWq1GqmpqQFzrenp6RgzZozNbaNHj0ZpaSmA7lid/Q6npaWhurra5rjBYEBdXZ2srhUA7r//fsvoy7hx43DjjTfid7/7nWWELdCu18xT1+Uvv9dAd+Jy8uRJrFmzxjLqAgTOtf7www+orq5GVlaW5bXq5MmTuO+++zB06FAAgXOtnsTkxYXQ0FBMmTIFa9eutdxmMpmwdu1azJw5U8LI+k4IgSVLluCzzz7DunXrkJ2dbXN8ypQpCAkJsbnWkpISlJaWWq515syZ2Lt3r80Tyfyi0vMNVEpz5szB3r17UVRUZPmaOnUq5s+fb/l7oFzrWWed1WvJ+6FDhzBkyBAAQHZ2NtLS0myutampCVu2bLG51oaGBuzYscNyzrp162AymZCfn++Dq3BfW1sblErbly6VSgWTyQQg8K7XzFPXNXPmTGzcuBF6vd5yzpo1azBq1CjEx8f76GpcMycuhw8fxnfffYfExESb44FyrTfeeCP27Nlj81qVkZGB+++/H6tXrwYQONfqUVJXDPuDjz76SGg0GvH222+L/fv3i9tuu03ExcXZrELxB3fccYeIjY0VGzZsEKdPn7Z8tbW1Wc65/fbbRVZWlli3bp3Yvn27mDlzppg5c6bluHn58EUXXSSKiorEqlWrRHJysuyWD9tjvdpIiMC51q1btwq1Wi2eeOIJcfjwYfH++++LiIgI8d5771nOefLJJ0VcXJz44osvxJ49e8Tll19ud4ntpEmTxJYtW8SmTZtETk6O5EuH7bnpppvEoEGDLEulV6xYIZKSksQDDzxgOcdfr7e5uVns2rVL7Nq1SwAQzz77rNi1a5dlhY0nrquhoUGkpqaKG2+8URQXF4uPPvpIRERE+HxJrbNr1el04uc//7kYPHiwKCoqsnm9sl5NEwjXak/P1UZC+M+1+gqTFze99NJLIisrS4SGhorp06eLn376SeqQ+gyA3a+33nrLck57e7v4zW9+I+Lj40VERIS48sorxenTp20e58SJE+Liiy8W4eHhIikpSdx3331Cr9f7+Gr6rmfyEkjX+tVXX4m8vDyh0WhEbm6ueOONN2yOm0wm8fDDD4vU1FSh0WjEnDlzRElJic05Z86cEddff72IiooSMTExYtGiRaK5udmXl+GWpqYmcffdd4usrCwRFhYmhg0bJv7whz/YvKn56/WuX7/e7nP0pptuEkJ47rp2794tZs+eLTQajRg0aJB48sknfXWJFs6u9fjx4w5fr9avX295jEC4VnvsJS/+cq2+ohDCqi0lERERkcyx5oWIiIj8CpMXIiIi8itMXoiIiMivMHkhIiIiv8LkhYiIiPwKkxciIiLyK0xeiIiIyK8weSEiIiK/wuSFiIiI/AqTFyIiIvIrTF6IiIjIrzB5ISIiIr/y//CLYaEmrUcyAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.plot(total)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb368900-34b3-4c36-a916-62ed3a97173b", "metadata": {}, "outputs": [], "source": [] diff --git a/hyena_test/simple_hyena_predict_more_than_one.ipynb b/hyena_test/simple_hyena_predict_more_than_one.ipynb new file mode 100644 index 0000000..f4f18d1 --- /dev/null +++ b/hyena_test/simple_hyena_predict_more_than_one.ipynb @@ -0,0 +1,850 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 32, + "id": "6c6e33cb-72f9-42fa-936a-33b5fe338d15", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 1024, 128]) torch.Size([1, 1024, 128])\n", + "Causality check: gradients should not flow \"from future to past\"\n", + "tensor(-6.7998e-10) tensor(0.1017)\n" + ] + } + ], + "source": [ + "# %load standalone_hyena.py\n", + "\"\"\"\n", + "Simplified standalone version of Hyena: https://arxiv.org/abs/2302.10866, designed for quick experimentation.\n", + "A complete version is available under `src.models.sequence.hyena`.\n", + "\"\"\"\n", + "\n", + "import math\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from einops import rearrange\n", + "\n", + "\n", + "def fftconv(u, k, D):\n", + " seqlen = u.shape[-1]\n", + " fft_size = 2 * seqlen\n", + " \n", + " k_f = torch.fft.rfft(k, n=fft_size) / fft_size\n", + " u_f = torch.fft.rfft(u.to(dtype=k.dtype), n=fft_size)\n", + " \n", + " if len(u.shape) > 3: k_f = k_f.unsqueeze(1)\n", + " y = torch.fft.irfft(u_f * k_f, n=fft_size, norm='forward')[..., :seqlen]\n", + "\n", + " out = y + u * D.unsqueeze(-1)\n", + " return out.to(dtype=u.dtype)\n", + "\n", + "\n", + "@torch.jit.script \n", + "def mul_sum(q, y):\n", + " return (q * y).sum(dim=1)\n", + "\n", + "class OptimModule(nn.Module):\n", + " \"\"\" Interface for Module that allows registering buffers/parameters with configurable optimizer hyperparameters \"\"\"\n", + "\n", + " def register(self, name, tensor, lr=None, wd=0.0):\n", + " \"\"\"Register a tensor with a configurable learning rate and 0 weight decay\"\"\"\n", + "\n", + " if lr == 0.0:\n", + " self.register_buffer(name, tensor)\n", + " else:\n", + " self.register_parameter(name, nn.Parameter(tensor))\n", + "\n", + " optim = {}\n", + " if lr is not None: optim[\"lr\"] = lr\n", + " if wd is not None: optim[\"weight_decay\"] = wd\n", + " setattr(getattr(self, name), \"_optim\", optim)\n", + " \n", + "\n", + "class Sin(nn.Module):\n", + " def __init__(self, dim, w=10, train_freq=True):\n", + " super().__init__()\n", + " self.freq = nn.Parameter(w * torch.ones(1, dim)) if train_freq else w * torch.ones(1, dim)\n", + "\n", + " def forward(self, x):\n", + " return torch.sin(self.freq * x)\n", + " \n", + " \n", + "class PositionalEmbedding(OptimModule):\n", + " def __init__(self, emb_dim: int, seq_len: int, lr_pos_emb: float=1e-5, **kwargs): \n", + " \"\"\"Complex exponential positional embeddings for Hyena filters.\"\"\" \n", + " super().__init__()\n", + " \n", + " self.seq_len = seq_len\n", + " # The time embedding fed to the filteres is normalized so that t_f = 1\n", + " t = torch.linspace(0, 1, self.seq_len)[None, :, None] # 1, L, 1\n", + " \n", + " if emb_dim > 1:\n", + " bands = (emb_dim - 1) // 2 \n", + " # To compute the right embeddings we use the \"proper\" linspace \n", + " t_rescaled = torch.linspace(0, seq_len - 1, seq_len)[None, :, None]\n", + " w = 2 * math.pi * t_rescaled / seq_len # 1, L, 1 \n", + " \n", + " f = torch.linspace(1e-4, bands - 1, bands)[None, None] \n", + " z = torch.exp(-1j * f * w)\n", + " z = torch.cat([t, z.real, z.imag], dim=-1)\n", + " self.register(\"z\", z, lr=lr_pos_emb) \n", + " self.register(\"t\", t, lr=0.0)\n", + " \n", + " def forward(self, L):\n", + " return self.z[:, :L], self.t[:, :L]\n", + " \n", + "\n", + "class ExponentialModulation(OptimModule):\n", + " def __init__(\n", + " self,\n", + " d_model,\n", + " fast_decay_pct=0.3,\n", + " slow_decay_pct=1.5,\n", + " target=1e-2,\n", + " modulation_lr=0.0,\n", + " modulate: bool=True,\n", + " shift: float = 0.0,\n", + " **kwargs\n", + " ):\n", + " super().__init__()\n", + " self.modulate = modulate\n", + " self.shift = shift\n", + " max_decay = math.log(target) / fast_decay_pct\n", + " min_decay = math.log(target) / slow_decay_pct\n", + " deltas = torch.linspace(min_decay, max_decay, d_model)[None, None]\n", + " self.register(\"deltas\", deltas, lr=modulation_lr)\n", + " \n", + " def forward(self, t, x):\n", + " if self.modulate:\n", + " decay = torch.exp(-t * self.deltas.abs()) \n", + " x = x * (decay + self.shift)\n", + " return x \n", + "\n", + "\n", + "class HyenaFilter(OptimModule):\n", + " def __init__(\n", + " self, \n", + " d_model,\n", + " emb_dim=3, # dim of input to MLP, augments with positional encoding\n", + " order=16, # width of the implicit MLP \n", + " fused_fft_conv=False,\n", + " seq_len=1024, \n", + " lr=1e-3, \n", + " lr_pos_emb=1e-5,\n", + " dropout=0.0, \n", + " w=1, # frequency of periodic activations \n", + " wd=0, # weight decay of kernel parameters \n", + " bias=True,\n", + " num_inner_mlps=2,\n", + " normalized=False,\n", + " **kwargs\n", + " ):\n", + " \"\"\"\n", + " Implicit long filter with modulation.\n", + " \n", + " Args:\n", + " d_model: number of channels in the input\n", + " emb_dim: dimension of the positional encoding (`emb_dim` - 1) // 2 is the number of bands\n", + " order: width of the FFN\n", + " num_inner_mlps: number of inner linear layers inside filter MLP\n", + " \"\"\"\n", + " super().__init__()\n", + " self.d_model = d_model\n", + " self.use_bias = bias\n", + " self.fused_fft_conv = fused_fft_conv\n", + " self.bias = nn.Parameter(torch.randn(self.d_model))\n", + " self.dropout = nn.Dropout(dropout)\n", + " \n", + " act = Sin(dim=order, w=w)\n", + " self.emb_dim = emb_dim\n", + " assert emb_dim % 2 != 0 and emb_dim >= 3, \"emb_dim must be odd and greater or equal to 3 (time, sine and cosine)\"\n", + " self.seq_len = seq_len\n", + " \n", + " self.pos_emb = PositionalEmbedding(emb_dim, seq_len, lr_pos_emb)\n", + " \n", + " self.implicit_filter = nn.Sequential(\n", + " nn.Linear(emb_dim, order),\n", + " act,\n", + " )\n", + " for i in range(num_inner_mlps):\n", + " self.implicit_filter.append(nn.Linear(order, order))\n", + " self.implicit_filter.append(act)\n", + "\n", + " self.implicit_filter.append(nn.Linear(order, d_model, bias=False))\n", + " \n", + " self.modulation = ExponentialModulation(d_model, **kwargs)\n", + " \n", + " self.normalized = normalized\n", + " for c in self.implicit_filter.children():\n", + " for name, v in c.state_dict().items(): \n", + " optim = {\"weight_decay\": wd, \"lr\": lr}\n", + " setattr(getattr(c, name), \"_optim\", optim)\n", + "\n", + " def filter(self, L, *args, **kwargs):\n", + " z, t = self.pos_emb(L)\n", + " h = self.implicit_filter(z)\n", + " h = self.modulation(t, h)\n", + " return h\n", + "\n", + " def forward(self, x, L, k=None, bias=None, *args, **kwargs):\n", + " if k is None: k = self.filter(L)\n", + " \n", + " # Ensure compatibility with filters that return a tuple \n", + " k = k[0] if type(k) is tuple else k \n", + "\n", + " y = fftconv(x, k, bias)\n", + " return y\n", + " \n", + " \n", + "class HyenaOperator(nn.Module):\n", + " def __init__(\n", + " self,\n", + " d_model,\n", + " l_max,\n", + " order=2, \n", + " filter_order=64,\n", + " dropout=0.0, \n", + " filter_dropout=0.0, \n", + " **filter_args,\n", + " ):\n", + " r\"\"\"\n", + " Hyena operator described in the paper https://arxiv.org/pdf/2302.10866.pdf\n", + " \n", + " Args:\n", + " d_model (int): Dimension of the input and output embeddings (width of the layer)\n", + " l_max: (int): Maximum input sequence length. Defaults to None\n", + " order: (int): Depth of the Hyena recurrence. Defaults to 2\n", + " dropout: (float): Dropout probability. Defaults to 0.0\n", + " filter_dropout: (float): Dropout probability for the filter. Defaults to 0.0\n", + " \"\"\"\n", + " super().__init__()\n", + " self.d_model = d_model\n", + " self.l_max = l_max\n", + " self.order = order\n", + " inner_width = d_model * (order + 1)\n", + " self.dropout = nn.Dropout(dropout)\n", + " self.in_proj = nn.Linear(d_model, inner_width)\n", + " self.out_proj = nn.Linear(d_model, d_model)\n", + " \n", + " self.short_filter = nn.Conv1d(\n", + " inner_width, \n", + " inner_width, \n", + " 3,\n", + " padding=2,\n", + " groups=inner_width\n", + " )\n", + " self.filter_fn = HyenaFilter(\n", + " d_model * (order - 1), \n", + " order=filter_order, \n", + " seq_len=l_max,\n", + " channels=1, \n", + " dropout=filter_dropout, \n", + " **filter_args\n", + " ) \n", + "\n", + " def forward(self, u, *args, **kwargs):\n", + " l = u.size(-2)\n", + " l_filter = min(l, self.l_max)\n", + " u = self.in_proj(u)\n", + " u = rearrange(u, 'b l d -> b d l')\n", + " \n", + " uc = self.short_filter(u)[...,:l_filter] \n", + " *x, v = uc.split(self.d_model, dim=1)\n", + " \n", + " k = self.filter_fn.filter(l_filter)[0]\n", + " k = rearrange(k, 'l (o d) -> o d l', o=self.order - 1)\n", + " bias = rearrange(self.filter_fn.bias, '(o d) -> o d', o=self.order - 1)\n", + " \n", + " for o, x_i in enumerate(reversed(x[1:])):\n", + " v = self.dropout(v * x_i)\n", + " v = self.filter_fn(v, l_filter, k=k[o], bias=bias[o])\n", + "\n", + " y = rearrange(v * x[0], 'b d l -> b l d')\n", + "\n", + " y = self.out_proj(y)\n", + " return y\n", + "\n", + " \n", + " \n", + "if __name__ == \"__main__\":\n", + " layer = HyenaOperator(\n", + " \n", + " d_model=128, \n", + " l_max=1024, \n", + " order=2, \n", + " filter_order=64\n", + " )\n", + " x = torch.randn(1, 1024, 128, requires_grad=True)\n", + " y = layer(x)\n", + " \n", + " print(x.shape, y.shape)\n", + " \n", + " grad = torch.autograd.grad(y[:, 10, :].sum(), x)[0]\n", + " print('Causality check: gradients should not flow \"from future to past\"')\n", + " print(grad[0, 11, :].sum(), grad[0, 9, :].sum())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 368, + "id": "032ef08a-8cc6-491a-9eb8-4a6b3f2d165e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 1023, 1]) torch.Size([1, 500])\n" + ] + } + ], + "source": [ + "class HyenaOperatorAutoregressive1D(nn.Module):\n", + " def __init__(\n", + " self,\n", + " d_model,\n", + " l_max,\n", + " order=2, \n", + " filter_order=64,\n", + " dropout=0.0, \n", + " filter_dropout=0.0, \n", + " **filter_args,\n", + " ):\n", + " super().__init__()\n", + "\n", + " self.l_max = l_max\n", + " self.d_model = d_model\n", + " self.l_max = l_max\n", + " self.order = order\n", + " inner_width = d_model * (order + 1)\n", + "\n", + " self.dropout = nn.Dropout(dropout)\n", + " self.in_proj = nn.Linear(d_model, inner_width)\n", + " self.out_proj = nn.Linear(d_model, d_model)\n", + " self.fc_before = nn.Linear(1, d_model) # Fully connected layer before the main layer\n", + " self.fc_after = nn.Linear(d_model, 500) # Fully connected layer after the main layer\n", + "\n", + " self.operator = HyenaOperator(\n", + " d_model=d_model,\n", + " l_max=l_max,\n", + " order=order, \n", + " filter_order=filter_order,\n", + " dropout=dropout, \n", + " filter_dropout=filter_dropout, \n", + " **filter_args,\n", + " )\n", + "\n", + " def forward(self, u, *args, **kwargs):\n", + " # Increase the channel dimension from 1 to d_model\n", + " u = self.fc_before(u) \n", + " # Pass through the operator\n", + " #[B,1024,128] --> [B,1024,128]\n", + " u = self.operator(u)\n", + " \n", + " last_state = u[:,-1,:]\n", + " # Decrease the channel dimension back to 1\n", + " y = self.fc_after(last_state)\n", + " return y,last_state\n", + "\n", + "\n", + "if __name__ == \"__main__\":\n", + " layer = HyenaOperatorAutoregressive1D(\n", + " d_model=128, \n", + " l_max=1024, \n", + " order=2, \n", + " filter_order=64\n", + " )\n", + "\n", + " x = torch.randn(1, 1023, 1, requires_grad=True) # 1D time series input\n", + " y, last_state = layer(x)\n", + "\n", + " #import pdb;pdb.set_trace()\n", + " print(x.shape, y.shape) # should now be [1, 1024, 1]\n", + "\n", + " #grad = torch.autograd.grad(y[:, 10, 0].sum(), x)[0]\n", + " #print('Causality check: gradients should not flow \"from future to past\"')\n", + " #print(grad[0, 11, 0].sum(), grad[0, 9, 0].sum())" + ] + }, + { + "cell_type": "code", + "execution_count": 369, + "id": "80cde67b-992f-4cb0-8824-4a6b7e4984ca", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train Epoch: 1 [0/640 (0%)]\tLoss: 0.569402\n", + "Train Epoch: 2 [0/640 (0%)]\tLoss: 0.459675\n", + "Train Epoch: 3 [0/640 (0%)]\tLoss: 0.455738\n", + "Train Epoch: 4 [0/640 (0%)]\tLoss: 0.334769\n", + "Train Epoch: 5 [0/640 (0%)]\tLoss: 0.400367\n", + "Train Epoch: 6 [0/640 (0%)]\tLoss: 0.353075\n", + "Train Epoch: 7 [0/640 (0%)]\tLoss: 0.372000\n", + "Train Epoch: 8 [0/640 (0%)]\tLoss: 0.354896\n", + "Train Epoch: 9 [0/640 (0%)]\tLoss: 0.370209\n", + "Train Epoch: 10 [0/640 (0%)]\tLoss: 0.323172\n", + "Train Epoch: 11 [0/640 (0%)]\tLoss: 0.324925\n", + "Train Epoch: 12 [0/640 (0%)]\tLoss: 0.319851\n", + "Train Epoch: 13 [0/640 (0%)]\tLoss: 0.267247\n", + "Train Epoch: 14 [0/640 (0%)]\tLoss: 0.307432\n", + "Train Epoch: 15 [0/640 (0%)]\tLoss: 0.270438\n", + "Train Epoch: 16 [0/640 (0%)]\tLoss: 0.274847\n", + "Train Epoch: 17 [0/640 (0%)]\tLoss: 0.283570\n", + "Train Epoch: 18 [0/640 (0%)]\tLoss: 0.282435\n", + "Train Epoch: 19 [0/640 (0%)]\tLoss: 0.325277\n", + "Train Epoch: 20 [0/640 (0%)]\tLoss: 0.326561\n", + "Train Epoch: 21 [0/640 (0%)]\tLoss: 0.305762\n", + "Train Epoch: 22 [0/640 (0%)]\tLoss: 0.298317\n", + "Train Epoch: 23 [0/640 (0%)]\tLoss: 0.277244\n", + "Train Epoch: 24 [0/640 (0%)]\tLoss: 0.271200\n", + "Train Epoch: 25 [0/640 (0%)]\tLoss: 0.246922\n", + "Train Epoch: 26 [0/640 (0%)]\tLoss: 0.231166\n", + "Train Epoch: 27 [0/640 (0%)]\tLoss: 0.301950\n", + "Train Epoch: 28 [0/640 (0%)]\tLoss: 0.272427\n", + "Train Epoch: 29 [0/640 (0%)]\tLoss: 0.278342\n", + "Train Epoch: 30 [0/640 (0%)]\tLoss: 0.246955\n", + "Train Epoch: 31 [0/640 (0%)]\tLoss: 0.304109\n", + "Train Epoch: 32 [0/640 (0%)]\tLoss: 0.254574\n", + "Train Epoch: 33 [0/640 (0%)]\tLoss: 0.234942\n", + "Train Epoch: 34 [0/640 (0%)]\tLoss: 0.269669\n", + "Train Epoch: 35 [0/640 (0%)]\tLoss: 0.239477\n", + "Train Epoch: 36 [0/640 (0%)]\tLoss: 0.282838\n", + "Train Epoch: 37 [0/640 (0%)]\tLoss: 0.264821\n", + "Train Epoch: 38 [0/640 (0%)]\tLoss: 0.236031\n", + "Train Epoch: 39 [0/640 (0%)]\tLoss: 0.239743\n", + "Train Epoch: 40 [0/640 (0%)]\tLoss: 0.289715\n", + "Train Epoch: 41 [0/640 (0%)]\tLoss: 0.240897\n", + "Train Epoch: 42 [0/640 (0%)]\tLoss: 0.212328\n", + "Train Epoch: 43 [0/640 (0%)]\tLoss: 0.253719\n", + "Train Epoch: 44 [0/640 (0%)]\tLoss: 0.210382\n", + "Train Epoch: 45 [0/640 (0%)]\tLoss: 0.284465\n", + "Train Epoch: 46 [0/640 (0%)]\tLoss: 0.292590\n", + "Train Epoch: 47 [0/640 (0%)]\tLoss: 0.200158\n", + "Train Epoch: 48 [0/640 (0%)]\tLoss: 0.269204\n", + "Train Epoch: 49 [0/640 (0%)]\tLoss: 0.213730\n", + "Train Epoch: 50 [0/640 (0%)]\tLoss: 0.268548\n", + "Train Epoch: 51 [0/640 (0%)]\tLoss: 0.186853\n", + "Train Epoch: 52 [0/640 (0%)]\tLoss: 0.263178\n", + "Train Epoch: 53 [0/640 (0%)]\tLoss: 0.232999\n", + "Train Epoch: 54 [0/640 (0%)]\tLoss: 0.265539\n", + "Train Epoch: 55 [0/640 (0%)]\tLoss: 0.227278\n", + "Train Epoch: 56 [0/640 (0%)]\tLoss: 0.266715\n", + "Train Epoch: 57 [0/640 (0%)]\tLoss: 0.243662\n", + "Train Epoch: 58 [0/640 (0%)]\tLoss: 0.231188\n", + "Train Epoch: 59 [0/640 (0%)]\tLoss: 0.234178\n", + "Train Epoch: 60 [0/640 (0%)]\tLoss: 0.270513\n", + "Train Epoch: 61 [0/640 (0%)]\tLoss: 0.240779\n", + "Train Epoch: 62 [0/640 (0%)]\tLoss: 0.278103\n", + "Train Epoch: 63 [0/640 (0%)]\tLoss: 0.235614\n", + "Train Epoch: 64 [0/640 (0%)]\tLoss: 0.248226\n", + "Train Epoch: 65 [0/640 (0%)]\tLoss: 0.242527\n", + "Train Epoch: 66 [0/640 (0%)]\tLoss: 0.312065\n", + "Train Epoch: 67 [0/640 (0%)]\tLoss: 0.201206\n", + "Train Epoch: 68 [0/640 (0%)]\tLoss: 0.263827\n", + "Train Epoch: 69 [0/640 (0%)]\tLoss: 0.286222\n", + "Train Epoch: 70 [0/640 (0%)]\tLoss: 0.279344\n", + "Train Epoch: 71 [0/640 (0%)]\tLoss: 0.263106\n", + "Train Epoch: 72 [0/640 (0%)]\tLoss: 0.247235\n", + "Train Epoch: 73 [0/640 (0%)]\tLoss: 0.243001\n", + "Train Epoch: 74 [0/640 (0%)]\tLoss: 0.231101\n", + "Train Epoch: 75 [0/640 (0%)]\tLoss: 0.290737\n", + "Train Epoch: 76 [0/640 (0%)]\tLoss: 0.274286\n", + "Train Epoch: 77 [0/640 (0%)]\tLoss: 0.254139\n", + "Train Epoch: 78 [0/640 (0%)]\tLoss: 0.296462\n", + "Train Epoch: 79 [0/640 (0%)]\tLoss: 0.223839\n", + "Train Epoch: 80 [0/640 (0%)]\tLoss: 0.282157\n", + "Train Epoch: 81 [0/640 (0%)]\tLoss: 0.193232\n", + "Train Epoch: 82 [0/640 (0%)]\tLoss: 0.319076\n", + "Train Epoch: 83 [0/640 (0%)]\tLoss: 0.195717\n", + "Train Epoch: 84 [0/640 (0%)]\tLoss: 0.243448\n", + "Train Epoch: 85 [0/640 (0%)]\tLoss: 0.218913\n", + "Train Epoch: 86 [0/640 (0%)]\tLoss: 0.267013\n", + "Train Epoch: 87 [0/640 (0%)]\tLoss: 0.243178\n", + "Train Epoch: 88 [0/640 (0%)]\tLoss: 0.211878\n", + "Train Epoch: 89 [0/640 (0%)]\tLoss: 0.294469\n", + "Train Epoch: 90 [0/640 (0%)]\tLoss: 0.270843\n", + "Train Epoch: 91 [0/640 (0%)]\tLoss: 0.212928\n", + "Train Epoch: 92 [0/640 (0%)]\tLoss: 0.284125\n", + "Train Epoch: 93 [0/640 (0%)]\tLoss: 0.236006\n", + "Train Epoch: 94 [0/640 (0%)]\tLoss: 0.260203\n", + "Train Epoch: 95 [0/640 (0%)]\tLoss: 0.270325\n", + "Train Epoch: 96 [0/640 (0%)]\tLoss: 0.199420\n", + "Train Epoch: 97 [0/640 (0%)]\tLoss: 0.313867\n", + "Train Epoch: 98 [0/640 (0%)]\tLoss: 0.285310\n", + "Train Epoch: 99 [0/640 (0%)]\tLoss: 0.284724\n", + "Train Epoch: 100 [0/640 (0%)]\tLoss: 0.256413\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.optim as optim\n", + "import torch.nn.functional as F\n", + "from torch.utils.data import DataLoader, Dataset\n", + "import numpy as np\n", + "\n", + "def generate_sine_with_noise(n_points, frequency, phase, amplitude, noise_sd):\n", + " # Generate an array of points from 0 to 2*pi\n", + " x = np.linspace(0, 2*np.pi, n_points)\n", + " \n", + " # Generate the sine wave\n", + " sine_wave = amplitude * np.sin(frequency * x + phase)\n", + " \n", + " # Generate Gaussian noise\n", + " noise = np.random.normal(scale=noise_sd, size=n_points)\n", + " \n", + " # Add the noise to the sine wave\n", + " sine_wave_noise = sine_wave + noise\n", + " \n", + " # Stack the sine wave and the noisy sine wave into a 2D array\n", + " output = np.column_stack((sine_wave, sine_wave_noise))\n", + " \n", + " return output\n", + " \n", + " \n", + "class SineDataset(Dataset):\n", + " def __init__(self, n_samples, n_points, frequency_range, phase_range, amplitude_range, noise_sd_range):\n", + " self.n_samples = n_samples\n", + " self.n_points = n_points\n", + " self.frequency_range = frequency_range\n", + " self.phase_range = phase_range\n", + " self.amplitude_range = amplitude_range\n", + " self.noise_sd_range = noise_sd_range\n", + "\n", + " def __len__(self):\n", + " return self.n_samples\n", + "\n", + " def __getitem__(self, idx):\n", + " # Generate random attributes\n", + " frequency = np.random.uniform(*self.frequency_range)\n", + " phase = np.random.uniform(*self.phase_range)\n", + " amplitude = np.random.uniform(*self.amplitude_range)\n", + " noise_sd = np.random.uniform(*self.noise_sd_range)\n", + "\n", + " # Generate sine wave with the random attributes\n", + " sine_wave = generate_sine_with_noise(self.n_points, frequency, phase, amplitude, noise_sd)\n", + "\n", + " # Return the sine wave and the parameters\n", + " return torch.Tensor(sine_wave[:-500, 1, None]), torch.Tensor(sine_wave[-500:, 0]), torch.Tensor([frequency, phase, amplitude, noise_sd])\n", + "\n", + "\n", + "\n", + "# Usage:\n", + "dataset = SineDataset(640, 1025, (1, 3), (0, 2*np.pi), (0.5, 1.5), (0.05, 1))\n", + "\n", + "def train(model, device, train_loader, optimizer, epoch):\n", + " model.train()\n", + " for batch_idx, (data, target, params) in enumerate(train_loader):\n", + " #data = data[...,None]\n", + " data, target = data.to(device), target.to(device)\n", + " optimizer.zero_grad()\n", + " output,last_state = model(data)\n", + " #import pdb;pdb.set_trace()\n", + "\n", + " loss = F.mse_loss(output, target)\n", + " loss.backward()\n", + " optimizer.step()\n", + " if batch_idx % 10 == 0:\n", + " print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n", + " epoch, batch_idx * len(data), len(train_loader.dataset),\n", + " 100. * batch_idx / len(train_loader), loss.item()))\n", + "\n", + "if __name__ == \"__main__\":\n", + " device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", + "\n", + " model = HyenaOperatorAutoregressive1D(\n", + " d_model=128, \n", + " l_max=1024, \n", + " order=2, \n", + " filter_order=64\n", + " ).to(device)\n", + "\n", + " optimizer = optim.Adam(model.parameters())\n", + "\n", + " # Assume 10000 samples in the dataset\n", + " #dataset = SineDataset(10000, 1025, 2, 0, 1, 0.1)\n", + " train_loader = DataLoader(dataset, batch_size=64, shuffle=True)\n", + "\n", + " for epoch in range(1, 101): # Train for 10 epochs\n", + " train(model, device, train_loader, optimizer, epoch)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc9f9031-5ee1-49f8-a70f-ad85ca015596", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 682, + "id": "90330622-8b44-4b45-8158-6840538f768c", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.metrics import r2_score\n", + "\n", + "def fit_and_evaluate_linear_regression(outputs_and_params):\n", + "\n", + " # Split the data into inputs (last_states) and targets (params)\n", + "\n", + " inputs = np.concatenate([x[0] for x in outputs_and_params])\n", + "\n", + " targets = np.concatenate([x[1] for x in outputs_and_params])\n", + "\n", + " \n", + "\n", + " r2_scores = []\n", + "\n", + " param_names = [\"frequency\", \"sin_phase\", \"amplitude\", \"noise_sd\"]\n", + "\n", + " \n", + "\n", + " # Fit the linear regression model for each parameter and calculate the R^2 score\n", + "\n", + " for i in range(targets.shape[1]):\n", + "\n", + " if param_names[i] == 'sin_phase':\n", + " # take the sine of the phase values\n", + " target_values = np.sin(targets[:, i])\n", + " \n", + " \n", + " #Not used at the moment\n", + " elif param_names[i] == 'cos_phase':\n", + " # take the cosine of the phase values\n", + " target_values = np.cos(targets[:, i])\n", + " else:\n", + " target_values = targets[:, i]\n", + "\n", + " model = LinearRegression().fit(inputs, target_values)\n", + "\n", + " pred = model.predict(inputs)\n", + "\n", + " score = r2_score(target_values, pred)\n", + "\n", + " r2_scores.append(score)\n", + "\n", + " print(f\"R^2 score for {param_names[i]}: {score:.2f}\")\n", + "\n", + " \n", + "\n", + " return r2_scores" + ] + }, + { + "cell_type": "code", + "execution_count": 683, + "id": "5eb62a22-cad8-43c4-b757-f36b6a01e9be", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_outputs(model, device, data_loader):\n", + " model.eval()\n", + " outputs_and_params = []\n", + " with torch.no_grad():\n", + " for data, target, params in data_loader:\n", + " data, target = data.to(device), target.to(device)\n", + " output, last_state = model(data)\n", + " outputs_and_params.append((last_state.cpu().numpy(), params.cpu().numpy()))\n", + " return outputs_and_params\n" + ] + }, + { + "cell_type": "code", + "execution_count": 684, + "id": "a95ee542-1c39-4f04-9184-e26c6983a018", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "R^2 score for frequency: -1.09\n", + "R^2 score for sin_phase: 0.84\n", + "R^2 score for amplitude: 0.96\n", + "R^2 score for noise_sd: 0.91\n" + ] + } + ], + "source": [ + "outputs_and_params = generate_outputs(model, device, train_loader)\n", + "\n", + "# Fit the linear regression model and print the R^2 score for each parameter\n", + "r2_scores = fit_and_evaluate_linear_regression(outputs_and_params)" + ] + }, + { + "cell_type": "markdown", + "id": "de65d0d2-b0c6-4ac5-a87f-70fa7f90480b", + "metadata": {}, + "source": [ + "# Autoregressive" + ] + }, + { + "cell_type": "code", + "execution_count": 519, + "id": "8ee139a6-aee5-4309-8685-bf0c28893279", + "metadata": {}, + "outputs": [], + "source": [ + "def predict_autoregressive(model, initial_data, n_steps):\n", + " model.eval()\n", + " predictions = []\n", + " current_input = initial_data\n", + " with torch.no_grad():\n", + " for _ in range(n_steps):\n", + " # Get the prediction for the next step and save it\n", + " next_output, last_state = model(current_input)\n", + " predictions.append(next_output)\n", + " #import pdb;pdb.set_trace()\n", + "\n", + " # Prepare the input for the next step\n", + " next_input = torch.cat((current_input[:, 500:, :], next_output[:,:,None]), dim=1)\n", + "\n", + " current_input = next_input\n", + "\n", + " return torch.cat(predictions, dim=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "272040d6-4dfb-438b-a432-744e950effd1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 594, + "id": "9f49cbf8-08e8-428a-af80-bcb2515a6327", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 525, 1])" + ] + }, + "execution_count": 594, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "initial_data = dataset[0]\n", + "initial_data[0][None,...].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 595, + "id": "c75464e3-e44d-4325-8804-29d8081e3a45", + "metadata": {}, + "outputs": [], + "source": [ + "autoregressive_out = predict_autoregressive(model,initial_data[0][None,...],3)" + ] + }, + { + "cell_type": "code", + "execution_count": 596, + "id": "90a37c56-59f3-49bc-bfbe-d9e126c42ed1", + "metadata": {}, + "outputs": [], + "source": [ + "total = np.concatenate([initial_data[0].squeeze().numpy() ,autoregressive_out.squeeze().numpy()])" + ] + }, + { + "cell_type": "code", + "execution_count": 597, + "id": "3d2ad0ee-7a0e-4b6f-8b40-dd565c8af84d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 597, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAGhCAYAAABceN/BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABehElEQVR4nO3dd3xUVfo/8M+kTXoCabRA6EgPCCGoSFNA1l5YRAWWxbKwFpQV3FXssKtf3dVVZP0JuCp2hVUUpCPSS+idhARIAgTSCKlzf39M5uZOMjW5debzfr2id2Zu7j0zE+Y+c85znmMSBEEAERERkQYCtG4AERER+S8GIkRERKQZBiJERESkGQYiREREpBkGIkRERKQZBiJERESkGQYiREREpBkGIkRERKQZBiJERESkGQYiREREpBnVApF58+bBZDLhySefVOuUREREpHOqBCI7duzAggUL0Lt3bzVOR0RERAYRpPQJSktLMWHCBHz44Yd49dVXvfpdi8WCc+fOISoqCiaTSaEWEhERkZwEQUBJSQlatWqFgADXfR6KByLTpk3D2LFjMXLkSLeBSEVFBSoqKsTbZ8+eRffu3ZVuIhERESkgJycHbdq0cbmPooHIF198gd27d2PHjh0e7T937ly89NJLDe7PyclBdHS03M0jIiIiBRQXFyM5ORlRUVFu91UsEMnJycETTzyBVatWITQ01KPfmT17NmbMmCHetj2R6OhoBiJEREQG40lahUkQBEGJky9duhR33nknAgMDxftqampgMpkQEBCAiooKu8ccKS4uRkxMDIqKihiIEBERGYQ312/FekRGjBiB/fv32903efJkdOvWDc8++6zbIISIiIh8n2KBSFRUFHr27Gl3X0REBOLi4hrcT0RERP6JlVWJiIhIM4pP35Vav369mqcjIiIinWOPCBEREWmGgQgRERFphoEIERERaYaBCBEREWmGgQgRERFphoEIERERaYaBCBEREWmGgYif2XaqAJ9vz9a6GURERABULmhG2hv3n60AgI4JkRjYvrnGrSEiIn/HHhE/lX2pTOsmEBERMRAhIiIi7TAQISIiIs0wEPFTJq0bQEREBAYiREREpCEGIkRERKQZTt/1EzmXyhAeEqh1M4iIiOwwEPEDBaUVuOEf6+zuMzFJhIiIdIBDM37gaH6J1k0gIiJyiIEIERERaYaBCBEREWmGgQgRERFphoEIERERaYaBiJ/irBkiItIDBiJERESkGQYifsDElWWIiEinGIj4KQYnRESkBwxEiIiISDMMRIiIiEgzDESIiIhIMwxE/ICjqbqcvktERHrAQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDSjaCAyf/589O7dG9HR0YiOjkZ6ejp+/vlnJU9JDuzMuqR1E4iIiBxSNBBp06YN5s2bh127dmHnzp0YPnw4br/9dhw8eFDJ01I9b/5yTOsmEBERORSk5MFvvfVWu9uvvfYa5s+fj61bt6JHjx5KnpqIiIgMQNFARKqmpgZff/01rly5gvT0dLVOS06YWFqViIh0QPFAZP/+/UhPT0d5eTkiIyPx/fffo3v37g73raioQEVFhXi7uLhY6eYRERGRhhSfNdO1a1dkZGRg27ZteOyxxzBx4kQcOnTI4b5z585FTEyM+JOcnKx084iIiEhDJkEQBDVPOHLkSHTs2BELFixo8JijHpHk5GQUFRUhOjpazWb6lJRZyxvc9874VNzWp5UGrSEiIl9XXFyMmJgYj67fquWI2FgsFrtgQ8psNsNsNqvcIv/EDBEiItIDRQOR2bNnY8yYMWjbti1KSkqwZMkSrF+/HitXrlTytCSzGouAi6UVSIoO1bopRETkYxQNRM6fP4+HHnoIubm5iImJQe/evbFy5UrcdNNNSp6WZDZx4XZsOnERn08dhPSOcVo3h4iIfIiigchHH32k5OFJJZtOXAQAfLrtNAMRIiKSFdea8WNf7czBwk2ZWjeDiIj8mOrJqqQff/lmHwBgVM8WaB0bpnFriIjIH7FHxE9J52xfqajWrB1EROTfGIgQERGRZhiI+CmV69gRERE5xECEWNyMiIg0w0DETz2/9IDXv8OAhYiI5MZAxE8VlzNBlYiItMdAhGBiVwcREWmEgQgRERFphoEIERERaYaBiI+yWATkFZV7uDfHZoiISBsMRHzU41/swaC5a7DyYJ7WTSEiInKKgYiP+nFfLgDgkU92adwSIiIi5xiIkMezZkycXkNERDJjIEJERESaYSDiQ9YdOY+cS2VaN4OIiMhjQVo3gOSx4dgFTF68AwCQNW+sxq0hIiLyDHtEfMTOrEuN/l1mfhARkVYYiBAREZFmGIiQx9hzQkREcmMgQkRERJphIEJERESaYSBCLFRGRESaYSDiIxhKEBGRETEQISIiIs0wECGHvSl7cwox8+u9OF9Srnp7iIjIf7CyKjlc9O72934DAFworVC5NURE5E/YI0IunThfKm4zp5WIiOTGQIRcEgStW0BERL6MgYgPen/9Ca/2twjA0bwSWCyMOoiISF0MRHzQP1Yc9Wr/uT8dxqh/bsTfVx5p8JjALhEiIlIQAxHCL4fyAQALNpzSuCVERORvGIgQERGRZhiIkEscmCEiIiUxEPEVnFtLREQGpGggMnfuXAwYMABRUVFITEzEHXfcgaNHvUukJG1Jc1UZ6hARkdwUDUQ2bNiAadOmYevWrVi1ahWqqqpw880348qVK0qelmQkcHCGiIgUpGiJ9xUrVtjdXrx4MRITE7Fr1y4MGTJEyVMTERGRAaiaI1JUVAQAaN68uZqn9QtyDpuwdojn8orK8c2uM6istmjdFCIiQ1Jt0TuLxYInn3wS1113HXr27Olwn4qKClRU1C2yVlxcrFbzqNYLyw7g1+MXxduMSRwTBAGPfroLKw9aa7CcvXwVT4zsrHGriIiMR7UekWnTpuHAgQP44osvnO4zd+5cxMTEiD/JyclqNc/wisurZDnOf7ecRuZF5vC4Ul1jQebFK2IQAgBvrz6GracKcCSvGFcrazRsHRGRsZgEFfrhp0+fjmXLlmHjxo1o37690/0c9YgkJyejqKgI0dHRSjfTUHKLruLfa09g0uAUdE6KQsqs5YqcJyHKjAsl1vfkjr6t8M/fpypyHqM4nl+Cm97e6HKfgSnN8dWj6Sq1iIhIf4qLixETE+PR9VvRoRlBEPDnP/8Z33//PdavX+8yCAEAs9kMs9msZJN8xrTPdmN3diG+230Wh18ZrXVz/Ia7IAQAtmddUqElRES+QdFAZNq0aViyZAmWLVuGqKgo5OXlAQBiYmIQFham5Kl93oGz1vyZq1XKDgPY1RHx86JpTOIlIpKfojki8+fPR1FREYYOHYqWLVuKP19++aWSp/UL6tX34MXXpsLJzJhTr9/S4D7OoiEi8oyigYggCA5/Jk2apORpiWRXYxGwdM/ZBvePuzYZAQEmXNuumd39T3+9FxXVTFolInKHa80YlElSOeSrHTkatsQ/LNl2GrO+2y/e7tMmBoM7xmHm6K4AgLfu62u3/w97z+G73Q0DFyIisqdaHRFSzl++3afYsZkWAVgsAp5fdlC8fUPneHwyJc1un7Zx4Xh7XB889eVe8b6C0goQEZFr7BExKK4Bo57l+3PF7f7tmmH+A/0d7ndnahuMvCZJvJ1XXK5424iIjI6BiEFV1TAQUcvyfXWByFePpCPS7Lwjce5dvdCmmXVG2I/7cjnThojIDQYi5BIvo0Dr2sAiyhyEwADXU5gToszisE1hWRXOXL6qePuIiIzMLwOR8yXl+GHvOWw4dkHrpuie9Bu9P1YRuXSlEh9tygQA3Ng1waPfaR8fgbDgQADADf9Yh3KFa70QERmZXwYih3NL8OfP9+DvPx/Ruim6d7lMnjVsjKi6xoJ+r6wSb/evN0XXlWtaRonbthL5RETUkF8GIrZv9hx2IFc2nyywuz35OtdLFEh9/IeB4vZn27JlaxMRka/xz0CkNhJhIiG5Il3R+MVbu3v1u1GhweL2BxtOytYmIiJf45eBSICfr5lCnpEmmg7qGNekY526UNrU5hAR+SS/DERsYYiFPSLkwsebswAAA1Oao1sL18tYu/OXb5QrOkdEZGR+GYhAHJrRthmG40cdSWWV1cgtshYku7lHkpu9HRvbu6W4XVpRLUu7iIh8jV8GIrZ1WhiHkDPnCuuGZWLCgl3s6dwdfVuL271axzS5TUREvsg/AxEmq5IbZwvryrPf1rdVo44x8ppEPDCoLQDg611ncLWS9USIiOrzz0Ck9v8MQ8iZhbVFzIZ1TYA5KLBRxzCZTHhyZBfx9je7z8jSNiIiX+KfgYjYJaJtO0iffjtxUay6mxQd2qRjxUeaxe2rlcwTISKqzy8DkYB6ccg/VhzBHxbvQI2FkQkBT36ZIW4/lJ7S5OM9NrQjAGBPdmGTj0VE5Gv8MhCxdYhkXryCXacv4f31J7H2yHn8duKitg1z4mIpS4SrSVqSvXNSZJOP1zc5FgDw84E8FPlxyXwiIkf8MhCRzkOdvGiHuF1ZbdGiMS59svU0rn11Nd5adUzrpvil4MCm/xOJlcy6Wb4/t8nHIyLyJX4ZiEgLq1bV6Hs45vmlBwAA76w5rnFLgBqLgK2nClBR7buzP86X1M2WGT+wrSzHDA8JEre/ZcIqEZEd/wxEJNsCM1Y9tizjHH7/n62Y/d1+rZuimDdWHBW3Hx/RSZZjSiv47jp9WZZjEhH5Cv8MRCRdInovJaLHZXG+231W6yYopvBqXQ5HpDnIxZ6eu6Zl08rDExH5Mr8MRAIkF3edxyGkspDanJCgAJPdCrpNOmZQAL59bLB4W5oMS0Tk7/wyEDHBcSSix94HHTZJl574Yg/uW7AFliZOwbbNUHprXF8ZWlVHOvvmvXUnZD22L8m6eAV7sjl8ReRP5Ol7NhiTXRyi7z4Rk8mk//EjHViWcQ4AcCi3GD2bsK7LtsxLAID4iBBZ2mUTHlxXnZUL4DmWXVCGoW+ut7vvvfv72S0eSES+xy97RIyEPSLeaUrMJk0kTYgyu9jTe0GSacCtY8NkPbYvOJpXgiFvrGtw/7Qlu/HvtdrPGCMi5fhlIGLXIyK5cFXV6K+OiB6Hi5xZsi0bm0/qsyicJ+6ev1nc7pTY9EJm9T0+ojMAIL+43M2e/ufhT3Y6fezNX45h4GuruUglkY/yz0BE0s9QLckpeObrfeL27O/2YebXe1VtlyMmg/SJZOQU4rnv9+P+D7dp3ZQmCwsOtJtZJRdbcLP/bJHsxza60wVl4vafhzecNn2+pALf7GINFiJf5J+BiJNrjG3svuhqFT7fnoOvd53Rvry6MeIQ3XzLb0rOT5821tySN+7tLVdz7AzuGIcAE3DwXDFyi64qcg6jigix5tB8/6fBePrmrjjyymhkzr0F9/ZvI+4z85t9XA+KyAf5ZSAS4OLb7onzpXYfdlr3BhskDkGYJBnTqBeLgiuVAIBWCuVwxEeaYXtp5q8/qcg5jKq8dnkF22sfWtsr9ca9ffDK7T3E/bacLNCkfUSkHL8MRFz1un+7+4zdWLTWORpan99TYSF1gcgVgy53X1BqDUTiZJ4x48h/t5xW/BxGUVVjEYPX0KDABo8/KFkB+YGPtuHkhVK1mkZEKvDPQMTN49Lv8656T9RglByRQEmVuLIK461Fc+pCKa5WWdsdFynvjBlneEG1sr3uAGAOdvyRtHjyAHH7X6s5i4bIl/hnIOLm2i5dGyRARz0ix/NLtGuIG9JCYnpcxdidpXvqytbLVdrdkedu6SZuj3p7o2LnMZLy2kDEZALMQY4/km7onCBu/2/vOVXaRUTq8MtAxF2fiDQvROseCenZb3p7I37W6TLy0rQQvReJc+RibX7I7xQuniVd0bfaoLk0cisqs67vYw4KcDpbKTDAhHl39RJv/3dLlhpNIyIV+GUg4qpHRBDse0S0vqjW/2B+7LPdGrXE3oF6U1D1kuDb2HMX1M6OSmvfXMbWNCTX+jW+orrGgptqe4bcDYPe3re1uP3x5iwlm0VEKvLLQMTdB570YsZZM45tPWU/e8Gi9QvVRAfOFgMA2sVFaNwS/3Ikr264sazSdW5RWEgg/n1/KgDrwoHVOixASETe88tAxJtkVc0vr3qNROqx6xHRsB2NyS0+cLYIZwutdT2ask6Np94Zb72Ydoj336DHYhEw6PU1+N27m7z6vWFdEwEAxeXV+M+vp5RoGhGpTNFAZOPGjbj11lvRqlUrmEwmLF26VMnTecxtsqrdMIPGQzOant25+kNGdsNZGr5mjTn1M5IKus3ClR866ZoUBQA4dfGK336rX74/F3n1iuDd0Dne7e9FmIPE4mcfsBYLkU9QNBC5cuUK+vTpg/fee0/J03jNXQKqnkYZlCg1Lof6rdJyaKapgY90lo8ar3d8ZF2dEulCe/7k4Lliu9ujeiThgwf6e/S7nz88CIC1V2TDsQuyt42I1KXcPEUAY8aMwZgxY5Q8RaN4M31X65hEp3FIg2nN0i/2Wr9m3qqsbfy3jw1W5XzSOiVnLl9Fmipn1Rfp38/eF25GjBc9Ub3bxIrbW04W4MYuCc53JiLd01WOSEVFBYqLi+1+tFAj6GMGCGDUoRm1W9M0tumjsSoMy9j8fkAyAGsg4m8EQcCJ89ZibiGBAV4FITZPjrSuZLziQK7dUCoRGY+uApG5c+ciJiZG/ElOTlbkPAFuqpQJOp6+qxf1m2V/MVD3NWtK4FNjEVBSu9hhbJh6gUibZtY1VbIKrqh2Tr149tt9+OVQPgDg3dpZMN6acn17BAaYkFVQJiYaE5Ex6SoQmT17NoqKisSfnJwcRc7j7tIu5zV147ELSHt9daPHsvUZhtS1q7LaggslFXa9SEZScMVaPyTABESrGIj0a9sMALD6cL5YWdQf5Fwqw1c7z4i3B6Q0rm5LVGgwWsWGAgBWHsyTpW1EpA1dBSJmsxnR0dF2P0pw18kg51TUhxZuR35xBSYu3N6o39dph4jYU3PLO79iwGurxa52QOOCZl7un3WxDACQFB2K4ED1/jkM6hCHuIgQlJRXN0jc9GU/H7CvDNyUWUpRZuvvvrr8sF8Fc0S+RleBiFpczZoRIOisOJc+IxFbgGQLQJbvq7vAqP3qNeV893+4FQCQW1TuZk95BQSY0KV2Gu+Zy2WqnltLr/90RNyecVOXJg093ndtG3F788mLTWoXEWlH0UCktLQUGRkZyMjIAABkZmYiIyMD2dnZSp7WLfd1ROq2tY5JLtaWHteb+sFcjUETBrVc7yU6zDpprbi8WrM2qKmi2r7X4vERnZt0vIfSU8TZN5tPFLjemYh0S9FAZOfOnUhNTUVqqjUhbcaMGUhNTcULL7yg5GndchWHbD1ZgDd+OSre1jpZVa/qB3PVOllrxhvS+iEPD+mg+vnDQ6yByPbMS6qfWwvvrT0hbn808domHy8gwIS37usLANjhp/VYiHyBooHI0KFDIQhCg5/FixcreVr3XEQie88UYaMksdQoF1VvDO4Y1+Rj1H8Jpd926wdvFouAt1cdw53v/4bd2fJfMBpb0EwaACi96q4jJbU9IT/sPad5BV81SEuyD++WKMsxr2lpzSPbm1PIPBEig/LLHBF3i95J+eLl4Z+/79vkY9R/CfOL64aQ6l9Tv99zFv9acxx7sgtx1/ubm3xuuRSXV4nbnROjVD//xMHtxG3bFGJfVVltQXmV/BVsW0SHituPf75HlmMSkbr8MhDRZ/qneiLNTS+o665MvtSx8yXud9LA0dqVX0f1SEJY7folarqhc4K4bsql0krVz6+my2V1z+/nJ26Q7bi2PBvAP2uyEPkC/wxEvOkR8cEuc296hJxycQi1X7LGnu5fa44DACqqtVt4LrH2G/05Hy/K9fKPh8Rt23CKHEwmE5ZOuw4AcCy/FOdL1J39RERN55+BiBf7ahmH6Ll0tatgxmgJvr1ax2h27s6JkQCAo/n67DWSi3R6t9yuaVk3rLbyYL5i5yEiZfhnIGKAsZn84nJ0eO4nh4/lXGpa3QmFO0Q05WkPliAICKktYDZugDJLCXiiVay11PuFEn1O05aD0r2K5qBAPHpjRwDAsTzfDuiIfJF/BiJeXEa16hFZ+Fum08de/N/BJh1bjqEZV4eo/5p583o3RmPeo6tVNeKqu83CQ2RukefiIqznvnTFd3NEpDOlnv9dd0XOYetZOq7TfCQics4/AxEvnrVWwwxhwc6TJ5s6w0KOsECWPJMmaOqw1cWSugt/uAaJqjYtYqw5Il/syPHZYOS57w6I21Oub6/IOTonWQORracu2U2/JyL9889ARIFjnrpQitnf7cNpmTL3Xc5saWJsJMfUSVeHUHoRsqe/2ovr/r4WJZLpt956/Iu6qZ5arnB8rWTRtz8s3qFZO5TUISFC8XN0TIgUt19dfsjFnv7hp/25itTsIVKCfwYiXs2a8Wy/cf/Zis+352DSInkuJkouwKb0ZfddSQVNJXy7+wxyi8rFBMjG9Fpl5BTK3KrGaR8fgahQa9ApXTjQV1y6UomfD1gD03/JUL/GmQhzENrHWwOets2VD3z0av3R8+jxwgr86bPduOv9zUiZtRxbTxWguLwKS/ecRamP16shY/LPQMSLfT29xNmSDTMvytMjEhepXN6CHB0AT3yRgZ1ZzkuTf7/njNPH5BIQ0PgnEhxo/d3b+7aSqzmN9tHEAQCAqhrtphEr5bOtp8Vtaa+FEp4caV27ZvXhfL+84K47ch6TFu3AlUr7CrO//89W9H7xFzz5ZQYe+WSnRq0jcs4/AxEvrl9a1BH5bvcZTF+iXJVIuYYi7vlgi9PHnvpyryzncCXQwfPw5N06X1yOqhoBASbg1Tt6yt8wL6XEhwOwBiJGXTzQmfOS2UCdEpUNRHpKpmFvz/SvRfC+2pGDyR4M7f12ogCFZZU+WR+JjMs/AxFvZs0o2A5nZnzl+iJutDodSgms7RHx9jM1t8ha9KpFdCiiQoPlbpbX4iLMCDABFsG3Zs/syLqET2p7RG7r0wqhLhKw5SDtcTlX6D+FzTYcu4C/fLvP7r5HhnRwOkOp78ur0H72Tzhf7D+vEembfwYiXnQITFy4HTtcDEGQuqSzZRo7NHOltts+MrTppe7lEBhgQvMIMwD4VGXQd2or1wLAiGvkWeTOncnXpQAA/rb0gOsdfcQzX+/FxIXbxduhwQE4+fotmH3LNZhyfXv8/MQNmD6sEzbPGt7gdz/dlq1mU4mcYiDixpnLV3GviyEILShdl0PPqix1eRSNTRGxTX+OkGHNHbkkRFkDEV8qbJYkWZAuvUPTV3z2RNvm4eK2dEVoX7Qn+zK+2WWfi7XiiSFiTyFgLaf/zKiuaBUbhmvbNbPb9501xxWf4UbkCf8MRAx+IffnoZlKybowQY2MRJ6onborx+J/cvHFQGT/mSIAwP1pbcU1dZQ2Ia1uRWO5Esf1SpoTMmtMN2TNG4uUeOczhr55bDDWPTMUd/VrLd73yCe7cLHUd/7myJj8MxAxdhzi16pq6oKwxiTdni28Ki5HH6thRdX6EiJrAxEfuSiUVlSL6+f0VnEtn5CgAPRJjgUAZF7w3UBk3s9HUFhWV0fnofR2Lvau0z4+Aq/d0cvuvmtfXY1dpzn87I2qGgvWHTmPl344iM0nL/p875vS9POVUEWMQ9QlZ+AnneLamMz/Qsly9O3jwl3sqS5bj8iJfN+oJZIl6Y1QO+DrGB+BvTmFOHnBN17L+i5fqcQHG06Kt4++OhrmIM8TgcNCAq29J7OWi/fdPX8LsuaNlbWdvubUhVJ8vj0bH/5qv/zGot+yxO2R1yTh73f3QlztFwvyjJ/2iBg7FNmR5b8VEy2S4KO6NnHVm3jkYmldIBKkYNE4byVFWz+4Vh3yjdVjsyULM97UPUnVc/dqY+2B2eCDpd6raixIfWWVePum7kleBSHkvUtXKnH/h1sx/P82NAhC6lt9OB/9X12N1386zFlJXtDPJ7GKjB2G+Ddp0NGYmhsXJTkYQ7smyNEkWQzrap1VUlJR7RNj9lm1Sx3cmdraLnlSDaN7tgBgDdjzinzrYiAtEAcA70/o1+hjrZ4xBKN7tBBvT/3vTtYXqWfX6Uvo98oqbD5pX5cmNDgAn08dhHfGp4oLLkr9Z+MpDHx9DfIZjHjEPwMRhT8XN5+86FP1IPRE+jHpKBBx9zlqu8h3SoxE7zax8jWsiaRJhh+sP+liT2PILrD2iEhnsailZUwY+tfOELnz/d9UP7+Sdp6u6w3dPGt4k5aC6JQYhQ8e7C/eXnUoH4dzuXoxYB32PZxbjLvn28+YXD3jRmTNG4sjr4xBesc43NanFVbV3nfo5VENvtykvb4Gh84Vq9l0Q/LTQETZSOT+D7fh5rc3KHoOkgzNeDGLyDYrZZiOekPqW3XY2MMzZZXV4hBTO43ycK5NsQYiuT7UI2KxCPixdn0lwFqQTw4v395D3N5/tlCWYxrdsoxzGPOvX8Xbc+/qhaOvjnZZHTg8JAiLJw/EsVfHIFpSo+iWd371mSFXpfhlIKIGaS6CXgzpkoD5TejK1QNp13GjhmZqe0TidZhMZlsrpYOLKZhG0P2FlSio7RHUKhB5KD1F3PaV4YY9koUaFzzYv0lrLUk9lJ6C8QOTAfhmXo238ovL8eSXGeLtZdOuw/iBbT3OxQkJCsC+F0fhgwfqPmun/ncngxEXGIj4kRdv7Y4xvVpq3QzZ1DQiWfWCjgOR3rVJlr4yhRdQfqE7Z+Ii6mbq5Fy6qkkb5HS+pBx3z98MwPq3e7PMCcC39rEu/vjT/jz8tD/Xzd6+60pFNdJeXyPe/nRKmjgd3Fuje7bEwknXiren/ncnftx3rqlN9EkMRPyI0WcLAU1PVj1z2XpRatMsTK4mySYh0trV7ktFzbSq1RIaHIhuLaIAAF/uNH4p8x/31gUHd/dvLfu/5f6Sqqt/+mw3tpz0r0UDAWvP2R3v1eUUTUhri+s7xzfpmMO7JeH/PVQXjExfsocJrA4wECHDmvO/g5jvRWJnjUXA2dpAJFmDJEp3bLVELpZW2q2pYyR6Wj343mutww3HfaA2y9KMs+L2/QPbyn58c1AgFk0eIN4e/+FW2c+hZ4IgYPqSPTh+3vq38sSIznjtzl5ufsszI7sn4Zenhoi3015fg7LKalmO7SsYiPgRtftD3lt3QvFz/H3FEY9TVXOLrqLaIiAkMMBuHRS9iIu09h7UWARcLtNfjpEnXlimn8XmOiRYc21+OZRv2MAOANYdPY99teXyf/zz9WgXp0wO0bCuiXZJ3P50sfxxXy6W1w5JDe+WiKdu6iLr8bskReG5W7qJt7u/sNInpunLhYGIHwkPUbfw0Rsrj8r+YdaUvENbka3WzcJUr23hieDAAHH9m/VHjZk0+JlkRdcUjSvXtpP0etVfHM5IPtlSVzukR6toRc/197t7i9v3frDFbm0nX7Xr9GX8+XPr+lMjuiXio4nXuvmNxnl4SEe8LullufbV1ciRFP7zZwxEPPTA/9uG3CJjJ72ptfCYVFW1IGu2eFMW/LONe+txWMamtHZlYFtBMCP7fwp9oHtKWsPkpR8OatiSxiuvqsGm4xcBAMsfv17xPC9pafKD54rx7W7jBnCe+vOS3eL2vLt7K/oa35/WFuNqhwwB4Omv9/rMrK6mYCDioU0nLuJv3+un29koFm3OxInzyo7Re/oP+d211qGic4X6DSifudnaJXxWx2105rQkeOrTJgadEqM0bI19Cf/+Kc01bEnj7TtThMoaCxKizOjeUtneEAAIDDDhl6eGwBxkfe22nvLtpNVTF0pxrrbWzLy7eol5Wkr6+z29cU//NgCA7ZmX8PVO3w/23GEg4oWLjaiW6k1wXVpRjbdXHfP6HHIIC1Zm2OaXg/LOnW/sl4d9ZwrF7es7NS0TXkm23ho9B0vO2BaZCwww4etHB2vcGitbLYfiq1Vu9tSnV348BAAYkNJMtVlvXZKi8NFEa+LqsoxzuOyjVaIFQRBf30hzkJjcrIZ5d/USi6P95dt9mn3u6wUDEYV589Hx+k+H8a81xxVriytP3yxvcpa3BEGQaVqb40jlng/qSjU/d8s1MpxHGa1irdOKt566hJ1Zxlqa3TY1ekS3RIQE6eOjxRbYnbxQqqsZPZ7YmXUJ+89ak1QHqNyj07dtrLj9/Z6zznc0sNv+/RvW1eZifTJloKp5Y0GBAfjy4UFiBdZ/rTnuc+sieUMfnxYEANgtWUdCaatnDHG/kww8/eh/a9UxpL2+Bp/UW9TLk+N5cg5p0p1eLpKO2AIRwD54MgJb4l2bZvrJwemaFIVIcxBKyqsNt+bH7uy6zwO110WKNAeJhbze/OWoqudWw+aTF8Ug74bO8Uht28zNb8gvLtKMjybVTZm+7u9rcaXCf2YqSen3E9lHeNOdalExaUmt8Xtv8zde+p/rpEJfT+xKUmGMWim2CqbJzfVTLC4oMABp7a29CRuPG2sm0tZTdT1i/SQ9FGqZeXNXAEBZZQ2WbDN+UTip+z/cJm5rmVQ9IKU57rvWmi9SYxHQY85KzdqiJQYiCvOms89gPcdNVl5V0+C+MJWnGOtNUL3VVI0UeOVctvaIJOuoRwQA0jvGAQDeXXvcMK9neVUN1h45DwD45tF0TaoiS4dnnvt+P0rKjZlnU9/RvLoVhl/4XXeP15BRyj/u6YPWkp7QT7ZkadcYjTAQ8YYg4Ps9Z/DFdmW+HRi56JIzznp59uYUotvzK/BqbbKYTURIkMP9bRwOzXjwsg3vlgjAuvCfkZRVNgzW9Mo2NKO36dGjerQAAJRXWcTF+PTurdrkxfCQQNWHZWwizUH48uFB4u1/q1CgUGk1FgFT/7tTvD1xcIp2jZH46YkbxO3nlx30uzVpGIh4obJGwFNf7sWs7/ajwMOqeM6+yJRWVDc4hppDM2px9pTeWGkdd/5/mzLt7ndXdK0xL9GJ86Xit8uxvVp4fwCV2dZIAYBCg8z2OF1wBcXl1vFtva3jk9w8XPzGufbweY1b45n1R63t/OMNHTTNaUrrECduL9hwSrN2yGXbqQKxsOGyadfpprBhTFgwvn2sbqbZ9CV7UFFtnC8hTaXKX/h7772HlJQUhIaGIi0tDdu3b1fjtLK7UFKX1bzvbJFHWfgmJ4MzPeesRP9XV6NY0t1Zo2EgolTXr7Nn5KwwWaiDacSrDuXjnTXOu9Uf/KhuvNfRLq8tr+t10XOiqs1Pj9d9OzLK1EnbNMg2zcIQYXbdq6UFW6/IHDc5SHpw6kKpWHtn/ED1ppQ681fJLLNLBvl7dKToahWe+34/AGthscauqquU/u2a2fVA/WHxDofD175I8U/lL7/8EjNmzMCcOXOwe/du9OnTB6NGjcL588b4ZiJ1sbTuH+HkRTsw85u9TT6mdEEuiw9WU/Z2TD5A8heZV1SOorIqTP3vTry16hg2Hr8IR6GNbR0OZy6X1QV7I6+Rd/l0JQQEmNAlyVpjoMggPSKra3sabFN49eaG2lVUr1bV6H6a5Nurj8MiAOkd4tAyRvvepT/e0B4htblL/V5ZpXFrGm/Sou3IKrD2hjw6pKPGrXEsrUMcXrm9BwDgtxMF6Pb8CsPkNTWF4oHIW2+9halTp2Ly5Mno3r07PvjgA4SHh2PhwoVKn1px3+32YH69g46GN1c6ng6n5R+cUh2Uzp6Rs54i2/2Xr1Ri0Nw16PPyL+JjntQZcdSxc12nuu7lqNBgt8fQgwsl1mG7j+oNXemR9O/25doPUb0ZLPkbOHVRv6vxZl68gh/2WvMD/nB9e41bY2UymexePyNeGIuuVolfWO7q1xptNV4HyZUH01MwfVgn8fbERTs0bI06FA1EKisrsWvXLowcObLuhAEBGDlyJLZsaVgjoaKiAsXFxXY/Rlf/uphdUOY06csHc1UbXQn1WH5Jg/tMHhzP0eO2+yZfl9K4xmjA1otjy23Rs/O1QVOACfj9APmXqJeDOSgQQ2tXls0u0O9CY8sy6r7cSANorb19X19xe+Y3+7RrSCN9vTNHHEr/h2RhP716ZlRXNAu3fmnaeOwCtmcaq7ihtxQNRC5evIiamhokJdl3hyclJSEvL6/B/nPnzkVMTIz4k5ys/fio3K64WI3WF5NVMy96t3ibrUfDWRJZY16hktokSq2n6XkjVYO6EY1lW4+kc2KUrnNwbIvgHcvXb4+IbZZUtxZRCHczg0xNzSJC0DEhAoB1JWMjzfA7V3gVry4/DAB47c6eDabI69WmZ4eL2/ct2IL3fGDWkjO6ekdmz56NoqIi8ScnJ0frJjVZ/aECV7GGloGIBmUKHLI1I0DGbHZbtVYjrZ787vhUANZehqIyfeeJ2OoyDGyv74Xl0mtngKw5Iu/6R3LKqg3cxw/UX8/S/0l6RYb933rN2uEtaS7feJ322DkSYQ7C7udvEm+/sfIoDpx1nQ9nVIoGIvHx8QgMDER+vv0//Pz8fLRo0XAapdlsRnR0tN2P0TnLhXBEz18ybu/bSp2u4tqIKMBBZGQymRo91ANYa5cYRZtm4WgWHgyLAOTJsgaPcg7lWodQO9R+Y9arwR2tCaunC8p0WUr7amUNfjlk/axsp8Mchj5tYpBYW/n3dEGZIXpFzhVexW8nrD12d6a2lvULjhqaR4TgFknJgd+9uwkbjhmrQrAnFA1EQkJC0L9/f6xZs0a8z2KxYM2aNUhPT1fy1LpVf9qqtIqenodmpg3rhM/+OMj9jjIJdBSIwPm0X2eqa+qmIv3dAGPDUrHhIQD0PXOmsKwS62sXDlN7YTZvxYQHI6p2avFbOlzt9LNtdesstYvTX1BnMpmw4sm6Naqk7dWrwfPWituv3dlTw5Y03vsT+mNs75bi7YkLt4vFA32F4kMzM2bMwIcffoiPP/4Yhw8fxmOPPYYrV65g8uTJSp9aFdIaFY64G/JYmnFO/Gah5eqg7r4nBKn8TUKuoaLc2qmawYEmXKvzC2V9tpU5t2cWaNwS56QLovVsHaNhSzxTUtsT8tGmTF3N/iivqsFCyQypFB32iADWb+h/Hm6d0fH8soM4V6jf4U5p2966r4+ucm689a9xfe1u3/CPdfhpf642jVGA4oHIuHHj8Oabb+KFF15A3759kZGRgRUrVjRIYDWqD391Pb2y/vXU4awOF495yzYz5Pa+rZp0nD/WmzoYrFKCl+31cpSsajJ5MGum3m3blL2uLaJ0U0XRU3tr2/7mL/r79m6zxiCVSm3mT+gnbucXe1YdWQ1fbM/Gudqg+de/DNNkbRlPTZKURf9rbYEwPXp/fV1y552prTVsSdMFBQYga95YvHd/3d/vnz7bjU3HL2rYKvmocnWZPn06Tp8+jYqKCmzbtg1paWlqnNYwbN/MGtsj0q1FFIZ0ScCCB/vjhd91x/pnhuL1O3t5dYz6H3zt6433q30Rd5Qj0hhrDlvH3Hu01P+39fr0mLBYny0BdIpOal64M6ZXS0TULiNwOE8f5QEEQcCLP1h7Vkd0S9TdWj31xUWacU9/64qx645ewKkL+puFVF1jEcv5Pziona4DO2+M7d0Sc27tLt5+4KNtSJm1HIVlja94q4eeQV3NmvFFnvwDsMUfjc0RSYgy479/GIhRPVrAZDIhJT5Ctgu5TVCgOv+Q66bvOn7c25cos8A6C8FI02FtHh7SQdzWctjOFdsSBZ0SIzVuieeG11bXPZLbsFaNFq5IFjZ0tMSBHv1ldFdxe8RbGzRsiWMLNp7CuaJyRIcG4a9jr3H/CwYy+br2uD/N/ktK35dXYe7Ph71OID5beBW/e3cTVhzI1TQgYSCiMM+GZgSnj+lFcIC6QzPOAilvk1VtsyPa6vxbpiO2GQoAsO9MoXYNcUIQBLG0e2yYMSrWAnWLCh7RSY/IhqN1syCeM8hFMzEqFEnR1r9PQdDXGjQV1TXiopp/GtbJMMGdN16+rUeD5NsFG06hw3M/YdqS3ShzVa/KIuDfa48jZdZyXDdvLQ6eK8Y/Vx/X9MuOcbN3dGTU2xub9PtCE3tE5OCuA0WtHhEbRz1J3nbynLlcJhav0uNCbO5EmIPQLDwYl8uqdDlzZr3kAlp/KE/PrmlpDUSWZZzDW/f11Tx3aNqS3eK2bZVgI3h/Qj/cPd9aIfubXTl4WCfrtzz2qfX1DA0OMMyQobeCAgMwIa0dJqS1w/d7zuCpL+tqpSzfl4vl+3IRGGBCq9hQRJmDxSn2zrx5bx9NC72xR0QGRx2UIxfVL2jm4Bt9RbWl9jH9Ui1Z1UW0YYJ3dUS+3nlG3DbqtyLbkMfVSv2twrlX0kvTJTFKu4Z46ZqWdfWJtC6dLZ3ZMTG9nYYt8V7/ds3xUG2bX//piC6GD89cLhOXRXhqZBfVPre0dGdqG/w2a7jY02dTYxGQc+mqyyAkMcqMY6+O0XzGm++/Sxrz5LvW019lANBH0pBN/aaoPX1XDtKnEBtunKEDqXOF1pkUtuqwemKbdfLEiM6GKhQlXdF21SFtq6xKF3K8oXOChi1pnLv7tRG3bRV2tfTOmuMArDlhj9yojx4aNbSODcOKJ4fgt1nDPapw/MY9vbHrbyOx/a8jdbEsg/H6qw3OUayxWoEpkN7mUri7jKjVfW07i6OgzJOhGemv2fJDOiVGIik6VIbWqe9s7TfmzSf1V0vElrdixNd2/MBkfL49Byc1nvHxiyQQGtYtUcOWNE6f5Fhx+9TFUnRvpV017JxLZfiqthf0wUHG6l2SS+vYMHz1iLVYqMUiICDAhLLKaphgQlhIIC6WViAuIkR3s4i0D4V8nDdveGP/OBr7e9//abCLY8pzDrl502lUXJtXcVc/Y9cQ0KPDucU4eM7a5WtLWjSSW3pZK1UeySvWrCeyqsaC+etPAgBmjuqqea5KY43uYS1BrnXC6qLfssTtm3s0XELE39h6KcNDghBWO2U9PtKsm89yKQYiCmuw6J2LfRv7gdjYPNLUts3qbujkj7OpzZC+hrappdGhxhyWAYAPHujnficNfLOrLv/GiDOSrmkZDZPJOrx0RKMhBVsgBwDjBhh3pfG4SOtSBPkarol0paIaC3+zFpf8v3v7INKAyen+jIGIypT49mXUb1KO2BYJdPYqeTPkZJsxE22gqaX1DelSlzdQUq6fmTPS2R1GqiFiEx9pFoux7chSP2HVYhFwx3u/AQBu6p6E+Ejj9SrZdEmyJkl+ti1bs4TVHnNWitu3NbGqNKmPgYjC6ocISvw7lbt4GaDfmiaelng/kleMzNol1aNCjfvtKDwkSEwmO3nhisatqWOb6XV3vza67Or1RFp7ayCyfJ/6a3ZIZ9r1bGW8qr9S4wYkIyYsGIVlVZoshHfwXJG4PeX69n4xU8bX+O07NrijCkvao2FuhV57RNS+lDjt2ahtiKOXyZsL3u7TheJ2cjPj1GZwpLL2oj+jdnaVHtjqmkSHGTfIu7GrtbdpW+Yl1Xubsi7WBZX1q2QaTWhwICbUPod5Px8Rk8TVUF5Vg7HvbAIAhAQGYOaorm5+g/TIbwORT6ekYf+LN6t+Xmc9Il/vzGl0b4mjqZOhQYF2+RY9vMxmV/JLbmODMRPc11qxHTq3yDrbpHVsGDoZqMaFK6d00iNSWlGNDzZYkyxbxhhvxoyNdIVbWxEstRyo/RZ/Z2prJEQZd1jG5s/DO6NlTCjKKmvEBFw1LNmWLW5/P22wYesF+Tu/DUQCAkyIUiGJseHQjONL6cxv9jX6HIEOooaAABMOvTQau5+/Ce+MT8WnU7xbaFDJoZkdWZedPlb3TJrWgIycQgDAozd2cL2jATw+ojMAYGhXfdSZ2JNd9/6lxBmnomp9seEhYsBdUa1ewbgrFdV4b531Yt3PgGsgORIWEij+fX68OUuVmUhVNRa8/KN1scCEKDN6GHyIy5/5bSCilkv1VkX0dlEiTzgbmgkLCUTziBDc1qcVmkWEuDyGmsP8lxVcKdI25GM7R2uDD8sAdb1ZeinzLm2HEWtfSH08eSAA18Gx3H7cd07cvrWP7yRW/nm4NWAuqajGP1cfV/x8b/5yVNz+4uFBip+PlMNARGHS6+ai3zINk6yqpC935Dh9zNVTMZk87ye5UmH9hhsRYtwcBhvbgnJFZfoIRKYv2SNuGz0xMFFSAyXzojpDX//dYk3oHNIlAbHhrr8gGEkryUyqf6057nLhtaY6nl+CBRtOAQDMQQHomGC8mVtUx9ifIgbz0g+HFJlrL8e1wKRiuuraI+dRUu74Q0qcvtvYgK3290prE+aMuNhdfbaL1amLV3QTjPiKrkl1+UO/HMxT/HwXSipw8FwxTCZrvQtfs3X2CHF7yuKdipyjqsaCmyQLja6ecaMi5yH1MBBR2dNf73W/kwuO1nzRqo5IU85bUWVp1O95EqDszLqECyXWdVB8obBRjKQOyqLNmRq2xHoRsGlhwNLu9ZlMJjxRm4Mz9+cjds9PCT/stQ7LdGsR7RNJqvW1iAkVc5q2nCqwmx0kl8mLdojbiyYNQLIBC+qRPQYiBvPO+NQG9yk9NNPGSZ7FMzc3nCp3T/82DvZsyNn0XdtTcfSoyaN5M8A9H2wRt1sYeFaHTaLkgnXgbJGLPZU35l+/ituv3NFTw5bIZ+LgFHH7481Zip1HEAQxuXJUjyTFzqO1J2sDEQCY/vluWRNXf9qfi00nLoq3r+8cL9uxSTsMRAzGUcghR4+Iq/VCVjw5xOPjvKlxd3P9jzxfmM4XEGDC/AnWUu9KLJDoKUEQcOJ83SJx4SHGf20BoHlECGz/hPYrGOhJj22r6uqLAgJMeOm2HgCAA2eLsf7YBVmOW1VjwZ8+q5tmvWHmUMPnKJEV30UFKTGFzVHnR1N6RBY82B9/Ht4Jw13MfnA2vNGUjhhnOSlij4jDgmbA//1yrPEnNTBpros0GFCTrZqqjdErgkotePBaAMAhyfovcssqKAMAhAUHIs2HAxHAuvptVO3f7LTPdovrPjXF3J+OiNuP3NgB7Qw8dZzsMRBRkFpl0psSiIzq0QJP39zVEGW6TQA2nyxwuY8gABG139T1umBcY0RKytSXV6lX80JKmtczolsiYsKNu4ZPfX2TYxEYYMLx86XYd6ZQ9uNXVlvwl2+s+WGje/r+yrABASasnzkUbZuHo6yyBlMW73D/Sy6cLykXF7UDgNljrmlqE0lHGIgoSJk4xFGyqiIncqspoYvTHJEmzt7JKy7HlUrrhXpwJ98ZP05NjhW3H/tslyZtkBb9SuvQXJM2KCUhyoxera09PC//cEj24689ko/y2kDO2yrHRhUXaRZLv+/IutzoNX3Kq2rwx4/rZuBsenaYLO0j/WAgoqDTBfJnjDscmlFx1owSp7KVYwekyaqNC+PeXGktchQTFoxoFSrnqkXaY5Vz6aqLPZUjHZqZNLi9Jm1Q0pDaxMc9OYWyV1pdI8ntuaVXS1mPrWeP3NgRd6a2BgBMW7Jb7BXyVM6lMvzu3U3Yd6YIzcKDsf6ZoWjTjLNkfA0DEQU99WWG7MesHweYgwIwSZL1LxdnYYASM3TS566V7Vh5tXVafGHart7YhoSahQeLKwL7kidGdgEA1FgEWWcnFZVV4etdZwBY1z6SFv7yB3Nu7Y6era29QF/tPIO3JBVRXdlysgA3/GMdTpwvRURIIP7z0LVIiWdeiC/yvU8THTlfW8tCTtJvxtOGdcSBl0ahZYx6H2zSQKQpMYm7/BlHjz+/7KDHx2cgIj/btElfmInkiHT22RsrPbtYeuKz7afF7ed/53+5DbHhIVgyta4E+ztrTyBl1nKcLXTes/fljmyM/3Br3e+MT8WAFN8aDqQ6DEQUpESyav1rv1LT15zFGNLgQ81qrABwsdTzwC4u0ndKZ9tIa8goUSjKnYO1M0o6JfpuOe0HB7UDAGw9dQmFTVgTyaa6xiIGNTNHdcXonv4zLCMVHRqMAy+NsrtvyuIdWL4v1y75+vPt2UiZtRzPfrtfvO/Ne/tgxDW+W3eFAH5tNBi1Jrc4i6HUquLa1CAuLtL3qlbe0rMFHq/d3nj8gqrd1IIg4Jva4YWHhxh/RWNnnrvlGnyy1dqDsei3LDx1U5cmHe/+D7dBEKw1V+67NlmOJhpWpDkIu5+/CS//cBBLM87hSF4Jpi3Z7fJ31j8zlMMxfoA9IgpqbMKlnsk2NOPkfrmmEbfzwbLPQYEBeGCQdRbCmcvqJqxOkcxa6COZweNrwkIC0a2Fdf2ZFQfymlQLqKC0AtuzLgEA0to398mS7t5qHhGCf/4+Fd//abDL/YZ1TcCp129hEOInGIgoSJGhGY3Lfch1fouTF0eup/fo0I4yHUlf2tYGWOcVWDzRlbVH6mZ9+NJsJEe+fDgdwYEmHM0vwfFGFo/LLy5H/1dXi7f/89C1cjXPJ6S2bYZjr47BlOvb44FBbRFSO8Q8c1RXHH9tDBZNHqjqbEDSlt8Pzbw/oR8+2XIaW065LpTVGBZFckS0/ccp26wZJ69NU6fvAtaLta8mqybVLjSXXyx/IjRZxYQHI7VtM2zPvIR31hzHv+/3vjDev9Yct7vNUuQNhQQF4PnfdQcAvHpHL41bQ1ry+38dt/RqiX/c01vrZnhO4y8Jcn1JUXLQKvtSmYJH15atez9f5R4Rm6dGNi1nwijG1FY//XFfbqOm8h7JrSsVP8LF8glExEBEUcVXm76+Qn1ad1ba54g0vjXOhmZs1CqPbzQtantETl28guP5JaqcU1qY75Zevl+eHADu6le3ivTv3t2EKxXVHv9udY0Fu7MLxdvvTfCdpQaIlMBAREGVNRb3O3lJ6zVh5Bq3ddZrIcfRb+vTSoaj6JO0ZszrPx1W5ZxT/1uXqBrmIyvuuhMTFmzXk/HRpkwXe9c5c7kMnf76s3j7n+P6+mzdFSK5MBCB9gmgRhJgV0ek8Zz1eEi/STaWLy8qJg0EyirVWfzuWH5dwmZiVKgq59QDaU/GW6uOodSDXpF/rLAvhDa2t3/WDSHyBgMRg1ErZmoZ7fiCI9f0XWeKrlYhs4nFutS6QGvlmZuteRrbMi/h1IXGzerwRu821sXgbuqe5JOl3Z0JDQ7E+5JgpOeclS73LyyrxP/2nhNvX98pnkmqRB7gvxJoP9zhDbWaOuKaRDw5sjM+mmg/7VCJtWbqy7zYtIvrTd19uwrjjV3qhgxWHMxT9FxXK2vEwHDqDb5byMyZ0T3se9dSZi132DNSWW1B35dX2d0359buiraNyFcoFoi89tprGDx4MMLDwxEbG6vUafyOdPrudR2VW+beZDLhyZFdGpRWDlAhdD1dUOZRN7gj6R3iEBPm23UuUuLrirUVX23c6+SpX49fQEl5NaJDg8SFy/xJQIAJiycPsLuv55yVYvXV6hoLThdcQZe//Wy3z86/jUTnpCjV2klkZIoVW6isrMS9996L9PR0fPTRR0qdRhbG6Q+x9ohs/+sIZF0sw8D26i8CZTc0o9A5XvrhUKN/V41ASWtRkoJii37LxKwx3RQ718Of7AIAXJvSHOEhvlmbxZ2hXROx5I9puP//bRPve37pAezNKcTu05dxqt5Q4vRhnRDvg0sMEClFsU+Wl156CQCwePFipU7hl0ywJgxqlTQYqPNhLDWGjvRg0uAULN6c5XYadFNUVtfN+vLl2iyeGNwpHl8/mo57P9gi3mdbe0fqt1nD0TpWvdWwiXyBrr4/VlRUoLi42O5HDYa6dmndVumsGUO9cL7ljtTWAIAEBb95F16tW302JY5rfgxIaY6DL40Sk4Xr2/7cCAYhRI2gq0Bk7ty5iImJEX+Sk/17tUo9ipXkX+gxDvGXHhFbhdVzReW4fKXpy9U7UlhWV5Bv2jDfXLvHWxHmIEwf3hlHXhmNMT1bIC4iBDv/NhJZ88Yi0clMMyJyzatAZNasWTCZTC5/jhw50ujGzJ49G0VFReJPTk5Oo4/lDa3Xb/GG1m39v/v6anp+d/wkDkF8ZIi4/fbqY4qcI6/IWka+U2IkUts2U+QcRhUaHIj5D/THrudvYj4IURN5lSPy9NNPY9KkSS736dCh8VP8zGYzzGb+o3ZF6wtte50vy+0vPSLmoLrCZko85wslFXho4XYAdWXliYiU4FUgkpCQgISEBKXaohkjXbv01FQ9tcXGn1YOH94tEWuPnFdkKYF/SnpZdmRdkv34REQ2iuWIZGdnIyMjA9nZ2aipqUFGRgYyMjJQWqp8JUhfxgRRd/zn9emcGAkAWLItG1dlriZ7SZJ38sodPWU9NhGRlGKByAsvvIDU1FTMmTMHpaWlSE1NRWpqKnbu3On+l1VmpEsX4xDX/KlH5Maudb2TcvdaSAvK+fIigkSkPcUCkcWLF0MQhAY/Q4cOVeqUfkFX11kdRkU6bJJiBneMF3N2vFmm3h1BEPDr8Yviba4eS0RK0tX0Xc24uHi1iwt3/qCf0+M1X+tZRWqzBV4fb8mS7Zjf7T4rbv/x+vayHZeIyBEGIm70ah2jdRPs+NM3/sbwt9fn1AVrefGtp+QbmtmdfVnc7tKC66UQkbIYiMB12XIlS2g3jn6utHq86PvL9F1HcmQqw15dU/c336OV/y10R0TqYiACIC7SjN/1bunwsRqLvgIRP77OesbPXp93x6eK2ysO5MlyzHNFVwEA4we2RY9W+uoRJCLfw0Ck1r/v7wdzUMOXQ2dxiK6us3rMx/C3HpEOCXUF5i6VNb3U+/nicjFR9dY+joNzIiI5MRCRuL5TfIP7LDqLRFhHxLVm4cHud/Ih0h6L+etPNvl4/1pzXNxObsZEbSJSHgMRiTfv7dNgZc0aneWIKBmGDEixricysH1zz9qiw5hoxk2OV0b1F7tOX3a/kwufbcsWt1vEsLQ7ESmPgYhEs4gQTB/e2e4+veWIKNmaBQ9eixd+1x0fPNC/Scf5z4NN+/2miA0Pcb+Tj3nx1u7i9t3zNzf6OCfOlyCotiLcwPbNERzIjwciUh4/adwovlrlficVCQr20DSPCMEfrm+P5hGeXcy7JDme2nlzjxbImjcW13WKk7N55MSk6+xrfZRXNa7c++rD51FtERAfGYIvHx4kR9OIiNzyatE7f7T3TJHWTbCjh/6ZfS/ejPKqGhRfdV3Ns01sOIACdRrl5wa2b47tmdZaInlF5UhpxCrJpeXW93NMz5bMRSIi1bBHxGD0kLISHRqMxCj3+QO8lqlI8nfx+fZs5/u58O91JwAA4SEs6U5E6mEgYjg6iEQ8xEBEPaN7thC3F2w85XVu0ze7zojbB88Vy9YuIiJ3GIgYgPSCroceEZv6gUZ6h/o5IYxE1PJQejt0k5RjX7rnrIu9G3rm673idndWUyUiFTEQcSM0WPuXSFqCXmeTeOyE1evSZ4+IeoICA/DmvX3E26culnr8uxdKKuxuTx/eSbZ2ERG5o/1VVscmpLXFN48O1nzhuzDJMuxKzpqRG+MQdbWKDRO331vneXGzAa+tFrfv7tcG0aH+VRSOiLTFQMSF6zrFo2frGHygYV2M1rFh+GjSAM3Or1ev3N5D6yboTv1p155UBd5UW87dZuaorrK2iYjIHQYiLujhG/1vs4ajX9tY8bae+0Pqv15KDs1EmDnz3JENM4eK2/9cfczt/p9tO213m9VUiUhtDEQMQLqQm4FGZrA3R7kaLMw/caxdXF39kHfWnsChc8X4YMNJHM8vcbi/3ioHE5H/YSDiAa2veXazZnTcJ3Jj1wS72/vPKhiIaP6u6Je0RP8t7/yKeT8fwU1vb3S47y+H8sXt9c8MVbppREQNsH/bAEz2kYgu/f3uXrinf7Jq52OPiHOeltYvq7SvjNuYaqxERE3FHhEP6Omip6eedOnLMrpnSwQGqPdCsQS5c1GhwXj0xo4u93lt+SF0f2GleHtUjySlm0VE5BADEYPR89CMmhiGuDZrTLcG91XVWJBXVI7tmZfw4a+Zdo+9fmcvtZpGRGSHgYgH9JSPoNdkVU87KF66TZ5pt+wQcW94t0S7209+kYFBc9fgvgVb7O7/bdZwxEWa1WwaEZGIgYjB6CkOaUxbJg5OwbRhrocNPKGn4FCv3h7X1+728v25DvdrLSmERkSkNgYiHtDTt2+9Vlb15iUKDWr66q56ek/0KibMfYXUX/8yTIWWEBE5x0DEYPSUpNnYlsgRSunnVdC3JVPTnK6XtPv5m5DcPFzlFhER2WMgYhD39G+DXq1jMLijZ1MzfZ2O4jFdG9wxHltmjRBvj+nZAr/+ZRhOvX5Lg5LwRERaYB0RF2wXOz1c86Qrq+qR+j01enhXjKFZRAiy5o3VuhlERA6xR4QMScWSJUREpCAGIp7gRU9WcuTb6ilXhoiIGo+BCMmCAzNERNQYDEQ8wJoV8pKjM4MdIkREvoGBCMlC7cCAgQgRkW9gIOIBpS96r9zRE/GRxptK2dg8DVlyRNhLRUTkExiIuBAWos7s5gcHtdPtGjK6xTiEiMgnsI6IA7PHdMPh3GLc0CkegLLXvJYxoQD0tYZMY7CHgoiIGkOxHpGsrCxMmTIF7du3R1hYGDp27Ig5c+agsrJSqVPK5pEbO+Kfv09FgArFKpKbWUtsW/ygSyRIxteTYQ8RkW9QrEfkyJEjsFgsWLBgATp16oQDBw5g6tSpuHLlCt58802lTmtYfhCH4Ic/Xw8AEAzf/0NERHJRLBAZPXo0Ro8eLd7u0KEDjh49ivnz5xsuEFG0eFbtoY3eI+LJS3RNy2gZz8c+ESIiX6BqsmpRURGaN2+u5il1z3Y5NXgc4hXmkxARkY1qyaonTpzAu+++67I3pKKiAhUVFeLt4uJiNZrmlpKXTdsXe8HHI5HHR3TWuglERKRDXveIzJo1CyaTyeXPkSNH7H7n7NmzGD16NO69915MnTrV6bHnzp2LmJgY8Sc5Odn7Z6SRTc8Oa9Tv2XoHLAaMQ7wJ0CYNThG35cgRYZ8KEZFv8LpH5Omnn8akSZNc7tOhQwdx+9y5cxg2bBgGDx6M//znPy5/b/bs2ZgxY4Z4u7i4WBfBiCfpCG1qZ7809ti+nsDJwIGIiBzxOhBJSEhAQkKCR/uePXsWw4YNQ//+/bFo0SIEBLjugDGbzTCbzd42ydBMYrKqtu1oKnfBmty5pZXVFnkPSEREmlAsWfXs2bMYOnQo2rZtizfffBMXLlxAXl4e8vLylDqlYlRJrjR4IOLIg4Paidtyv4ZBgexjISLyBYolq65atQonTpzAiRMn0KZNG7vHfD0x0xt1OSK+95o8NrQjPtl62npD5rihdWyYvAckIiJNKNYjMmnSJAiC4PCH6ph8pI6II9LhGLmHZgQAKhS+JSIihXHRO09ILnhyFuWS8r0wRHkBLGpGRGR4DESaoH18hGzHMnqHiKMcEOl9cj8/E+TvZSEiIvUxEPGA9IInHVqSY5jJUanyvsmxTT6u2hxNP3YWKMgRlAiwD3TiI0NwZ2rrph+YiIhUpVplVV8kx5d8fqkHggNNqKppxKspefG2PTcSgUwaISIyHPaIeEB6eZN+m5fjm70vDy9IczhcPc/QoECvj22CfbIqgxAiImNij4iXereJwdH8EgDyzHQx8uXTXRCVEGXGnamtEWAyITo02Ol+LWNDUZJf6v35Df3qERERwEDEI9I8jkdu7IhOiZEY2jURf1i8w+Nj3N2vDczBAViyLdvpsW2MkrfaIiZU3A52UjX37XF93R7ngwf6Y87/DuLX4xe9Or8v9yYREfkLBiJeMgcF4JEbOzbqd8ODnQ9BdEiIwKkLVxAT5rznQG/MQYHY9+LNCDSZENCEoZEOCZH4ZEoaUmYt9/h3rMmqRERkdMwRaQJvZ804uljb7lk0aQDu6d8G3z6WLkPL1BMdGowIszbxLOuIEBEZH3tEPODscuftQnWOLpy2u9rFReDNe/t4d0AfE2kOQmlFtUf7msT/EBGRkbFHpAkc1c5wJdDhq+1/V1Nnr9rKp4Z4dQz2iBARGR8DEQ/IUZhLgOCyR6TRB/Yh3i5kxziEiMj4GIg0gbfhgsNARJ6m+CW+dkRExsdApAm8Tlb1tEfEjy2ZmubxvhyaISIyPgYiTfDEiM4AgLv6ebbGieMckYb8aWDG9hraDO4Y7/HvMg4hIjI+zpppggcGtcP1nRPQrnm4+50FZ9N3/ftq+tRNXRr9u46KwRERkbEwEGkCk8mE9vERHu/PoRl58aUjIjI+Ds2oKMhBj4ijQCYxKrTBfdQQgzgiIuNjj4iXmnLxc9QjMn14pwb3vXpHT9RYLJg4OKXxJ9MzmaYn+/uwFhGRL2CPiIzeHZ+KLkmRmJDW1uHj9TtE/jb2GoSHNIwFW8SEYtHkgRjaNVGJZvqEkMCABq8nEREZD3tEZHRrn1a4tU8rLNyU2eAxAUBgvSvnlOvbq9QyY7mhc7zTlXjH9m6JuIgQpMRHMFmViMgHsEdERXGRZrvbfnshdfO859zaw+ljT43sgpdv7wkACA9xvpoxEREZAwMRlbSLC8foHi20boY+uMkRCQ70LEB7Z3wq2sdH4N3xqXK0ioiINMChGQ9Ih1Qc5XS48sXDg/DLwXw8emNHh3VEqKEgDyu/XdMyGuueGapsY4iISFEMRDxgDgrE2+P6oLLaguYRIV797qAOcRjUIU6hlvmmYAZsRER+g4GIh+5MbaN1E/xG/aRee/5UAJ+IyPcxR4RU5y6UcDU0Ex0aLG9jiIhIU+wRId1xlKzaPCIEf73lGiRGs+osEZEvYY8Iqc5dBkhQQMM/y1E9WuDu/hweIyLyNQxESHXuhmY8nb5LRETGx0CEdMdvC70REfkhBiJkCIxNiIh8EwMRlX3x8CB0iI/AZ39M07opmkliwikREdXirBmVDeoQh7V+Xg103IBkHM0rwQ2d47VuChERaYyBCKkuODAAr9zRU+tmEBGRDnBohgwhLJgr7RIR+SJFA5HbbrsNbdu2RWhoKFq2bIkHH3wQ586dU/KU5GOCA03o3SYG04d10ropRESkAEUDkWHDhuGrr77C0aNH8e233+LkyZO45557lDwl+ZgHBrXD/6Zfj2ZeLjZIRETGoGiOyFNPPSVut2vXDrNmzcIdd9yBqqoqBAdzzRByT+Aad0REPk21HJFLly7hs88+w+DBgxmEkMcERiJERD5N8UDk2WefRUREBOLi4pCdnY1ly5Y53beiogLFxcV2P+TfurSI0roJRESkIK8DkVmzZsFkMrn8OXLkiLj/zJkzsWfPHvzyyy8IDAzEQw895PRb7ty5cxETEyP+JCcnN/6ZkaH9b/p1mD2mG8Zdy78BIiJfZhK87Pu+cOECCgoKXO7ToUMHhIQ0TC48c+YMkpOTsXnzZqSnpzd4vKKiAhUVFeLt4uJiJCcno6ioCNHR0d40U1MLN2Xi5R8PAQCy5o3VuDVERETqKi4uRkxMjEfXb6+TVRMSEpCQkNCohlksFgCwCzakzGYzzGZzo45NRERExqPYrJlt27Zhx44duP7669GsWTOcPHkSzz//PDp27OiwN4SIiIj8j2LJquHh4fjuu+8wYsQIdO3aFVOmTEHv3r2xYcMG9noQERERAAV7RHr16oW1a9cqdXgiIiLyAVxrhoiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0BEAV2SorRuAhERkSEEad0AX3R953i8dV8fdE5kQEJEROQKAxGF3NWvjdZNICIi0j0OzRAREZFmGIgQERGRZhiIEBERkWYYiBAREZFmGIgQERGRZhiIEBERkWYYiBAREZFmGIgQERGRZhiIEBERkWYYiBAREZFmGIgQERGRZhiIEBERkWYYiBAREZFmdL36riAIAIDi4mKNW0JERESesl23bddxV3QdiJSUlAAAkpOTNW4JEREReaukpAQxMTEu9zEJnoQrGrFYLDh37hyioqJgMplkPXZxcTGSk5ORk5OD6OhoWY+tNV9+bgCfn9H58vPz5ecG8PkZmdrPTRAElJSUoFWrVggIcJ0FousekYCAALRp00bRc0RHR/vcH5yNLz83gM/P6Hz5+fnycwP4/IxMzefmrifEhsmqREREpBkGIkRERKQZvw1EzGYz5syZA7PZrHVTZOfLzw3g8zM6X35+vvzcAD4/I9Pzc9N1sioRERH5Nr/tESEiIiLtMRAhIiIizTAQISIiIs0wECEiIiLN+GUg8t577yElJQWhoaFIS0vD9u3btW6SW3PnzsWAAQMQFRWFxMRE3HHHHTh69KjdPkOHDoXJZLL7efTRR+32yc7OxtixYxEeHo7ExETMnDkT1dXVaj4Vh1588cUGbe/WrZv4eHl5OaZNm4a4uDhERkbi7rvvRn5+vt0x9PrcACAlJaXB8zOZTJg2bRoA4713GzduxK233opWrVrBZDJh6dKldo8LgoAXXngBLVu2RFhYGEaOHInjx4/b7XPp0iVMmDAB0dHRiI2NxZQpU1BaWmq3z759+3DDDTcgNDQUycnJ+Mc//qH0U3P53KqqqvDss8+iV69eiIiIQKtWrfDQQw/h3Llzdsdw9H7PmzdP8+cGuH/vJk2a1KDto0ePtttHr+8d4P75Ofp3aDKZ8MYbb4j76PX98+Q6INdn5fr169GvXz+YzWZ06tQJixcvVu6JCX7miy++EEJCQoSFCxcKBw8eFKZOnSrExsYK+fn5WjfNpVGjRgmLFi0SDhw4IGRkZAi33HKL0LZtW6G0tFTc58YbbxSmTp0q5Obmij9FRUXi49XV1ULPnj2FkSNHCnv27BF++uknIT4+Xpg9e7YWT8nOnDlzhB49eti1/cKFC+Ljjz76qJCcnCysWbNG2LlzpzBo0CBh8ODB4uN6fm6CIAjnz5+3e26rVq0SAAjr1q0TBMF4791PP/0k/PWvfxW+++47AYDw/fff2z0+b948ISYmRli6dKmwd+9e4bbbbhPat28vXL16Vdxn9OjRQp8+fYStW7cKv/76q9CpUydh/Pjx4uNFRUVCUlKSMGHCBOHAgQPC559/LoSFhQkLFizQ7LkVFhYKI0eOFL788kvhyJEjwpYtW4SBAwcK/fv3tztGu3bthJdfftnu/ZT+W9Xqubl7foIgCBMnThRGjx5t1/ZLly7Z7aPX986T5yd9Xrm5ucLChQsFk8kknDx5UtxHr++fJ9cBOT4rT506JYSHhwszZswQDh06JLz77rtCYGCgsGLFCkWel98FIgMHDhSmTZsm3q6pqRFatWolzJ07V8NWee/8+fMCAGHDhg3ifTfeeKPwxBNPOP2dn376SQgICBDy8vLE++bPny9ER0cLFRUVSjbXrTlz5gh9+vRx+FhhYaEQHBwsfP311+J9hw8fFgAIW7ZsEQRB38/NkSeeeELo2LGjYLFYBEEw9ntX/8PeYrEILVq0EN544w3xvsLCQsFsNguff/65IAiCcOjQIQGAsGPHDnGfn3/+WTCZTMLZs2cFQRCE999/X2jWrJnd83v22WeFrl27KvyM6ji6kNW3fft2AYBw+vRp8b527doJb7/9ttPf0cNzEwTHz2/ixInC7bff7vR3jPLeCYJn79/tt98uDB8+3O4+o7x/9a8Dcn1W/uUvfxF69Ohhd65x48YJo0aNUuR5+NXQTGVlJXbt2oWRI0eK9wUEBGDkyJHYsmWLhi3zXlFREQCgefPmdvd/9tlniI+PR8+ePTF79myUlZWJj23ZsgW9evVCUlKSeN+oUaNQXFyMgwcPqtNwF44fP45WrVqhQ4cOmDBhArKzswEAu3btQlVVld371q1bN7Rt21Z83/T+3KQqKyvx6aef4g9/+IPdYo5Gfu+kMjMzkZeXZ/d+xcTEIC0tze79io2NxbXXXivuM3LkSAQEBGDbtm3iPkOGDEFISIi4z6hRo3D06FFcvnxZpWfjXlFREUwmE2JjY+3unzdvHuLi4pCamoo33njDrutb789t/fr1SExMRNeuXfHYY4+hoKBAfMyX3rv8/HwsX74cU6ZMafCYEd6/+tcBuT4rt2zZYncM2z5KXSd1veid3C5evIiamhq7NwAAkpKScOTIEY1a5T2LxYInn3wS1113HXr27Cnef//996Ndu3Zo1aoV9u3bh2effRZHjx7Fd999BwDIy8tz+Nxtj2kpLS0NixcvRteuXZGbm4uXXnoJN9xwAw4cOIC8vDyEhIQ0+KBPSkoS263n51bf0qVLUVhYiEmTJon3Gfm9q8/WHkftlb5fiYmJdo8HBQWhefPmdvu0b9++wTFsjzVr1kyR9nujvLwczz77LMaPH2+3kNjjjz+Ofv36oXnz5ti8eTNmz56N3NxcvPXWWwD0/dxGjx6Nu+66C+3bt8fJkyfx3HPPYcyYMdiyZQsCAwN95r0DgI8//hhRUVG466677O43wvvn6Dog12els32Ki4tx9epVhIWFyfpc/CoQ8RXTpk3DgQMHsGnTJrv7H374YXG7V69eaNmyJUaMGIGTJ0+iY8eOajfTK2PGjBG3e/fujbS0NLRr1w5fffWV7H/0Wvvoo48wZswYtGrVSrzPyO+dv6qqqsJ9990HQRAwf/58u8dmzJghbvfu3RshISF45JFHMHfuXF2W2Jb6/e9/L2736tULvXv3RseOHbF+/XqMGDFCw5bJb+HChZgwYQJCQ0Pt7jfC++fsOmBEfjU0Ex8fj8DAwAYZxPn5+WjRooVGrfLO9OnT8eOPP2LdunVo06aNy33T0tIAACdOnAAAtGjRwuFztz2mJ7GxsejSpQtOnDiBFi1aoLKyEoWFhXb7SN83ozy306dPY/Xq1fjjH//ocj8jv3e29rj6d9aiRQucP3/e7vHq6mpcunTJEO+pLQg5ffo0Vq1a5XZZ9bS0NFRXVyMrKwuAvp9bfR06dEB8fLzd36KR3zubX3/9FUePHnX7bxHQ3/vn7Dog12els32io6MV+WLoV4FISEgI+vfvjzVr1oj3WSwWrFmzBunp6Rq2zD1BEDB9+nR8//33WLt2bYNuQUcyMjIAAC1btgQApKenY//+/XYfIrYP0e7duyvS7sYqLS3FyZMn0bJlS/Tv3x/BwcF279vRo0eRnZ0tvm9GeW6LFi1CYmIixo4d63I/I7937du3R4sWLezer+LiYmzbts3u/SosLMSuXbvEfdauXQuLxSIGYenp6di4cSOqqqrEfVatWoWuXbtq2rVvC0KOHz+O1atXIy4uzu3vZGRkICAgQBzS0Otzc+TMmTMoKCiw+1s06nsn9dFHH6F///7o06eP23318v65uw7I9VmZnp5udwzbPopdJxVJgdWxL774QjCbzcLixYuFQ4cOCQ8//LAQGxtrl0GsR4899pgQExMjrF+/3m5KWVlZmSAIgnDixAnh5ZdfFnbu3ClkZmYKy5YtEzp06CAMGTJEPIZt2tbNN98sZGRkCCtWrBASEhJ0McX16aefFtavXy9kZmYKv/32mzBy5EghPj5eOH/+vCAI1ilpbdu2FdauXSvs3LlTSE9PF9LT08Xf1/Nzs6mpqRHatm0rPPvss3b3G/G9KykpEfbs2SPs2bNHACC89dZbwp49e8SZI/PmzRNiY2OFZcuWCfv27RNuv/12h9N3U1NThW3btgmbNm0SOnfubDcFtLCwUEhKShIefPBB4cCBA8IXX3whhIeHKz5F0tVzq6ysFG677TahTZs2QkZGht2/RduMg82bNwtvv/22kJGRIZw8eVL49NNPhYSEBOGhhx7S/Lm5e34lJSXCM888I2zZskXIzMwUVq9eLfTr10/o3LmzUF5eLh5Dr++du+dnU1RUJISHhwvz589v8Pt6fv/cXQcEQZ7PStv03ZkzZwqHDx8W3nvvPU7fldu7774rtG3bVggJCREGDhwobN26VesmuQXA4c+iRYsEQRCE7OxsYciQIULz5s0Fs9ksdOrUSZg5c6ZdLQpBEISsrCxhzJgxQlhYmBAfHy88/fTTQlVVlQbPyN64ceOEli1bCiEhIULr1q2FcePGCSdOnBAfv3r1qvCnP/1JaNasmRAeHi7ceeedQm5urt0x9PrcbFauXCkAEI4ePWp3vxHfu3Xr1jn8e5w4caIgCNYpvM8//7yQlJQkmM1mYcSIEQ2ed0FBgTB+/HghMjJSiI6OFiZPniyUlJTY7bN3717h+uuvF8xms9C6dWth3rx5mj63zMxMp/8WbTVhdu3aJaSlpQkxMTFCaGiocM011wivv/663YVcq+fm7vmVlZUJN998s5CQkCAEBwcL7dq1E6ZOndrgi5pe3zt3z89mwYIFQlhYmFBYWNjg9/X8/rm7DgiCfJ+V69atE/r27SuEhIQIHTp0sDuH3Ey1T46IiIhIdX6VI0JERET6wkCEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTDQISIiIg0w0CEiIiINMNAhIiIiDTz/wGH+oXLwDrbCwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.plot(total)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb368900-34b3-4c36-a916-62ed3a97173b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50253cc9-254c-4933-94bd-36c28d0f97b0", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "895a5767-2865-477f-aa21-094a711bf033", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}