From 41d8647e07d15eabab9cfa3abecd16b523f03161 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Wed, 10 Apr 2024 03:30:19 -0400 Subject: [PATCH 01/62] h2o for kv cache compression Signed-off-by: n1ck-guo --- .../text-generation/h2o/generate_task_data.py | 74 ++ .../h2o/run_lm_eval_harness.py | 211 +++++ .../text-generation/h2o/tasks/__init__.py | 1 + .../text-generation/h2o/tasks/eval_harness.py | 108 +++ .../pytorch/text-generation/h2o/tasks/util.py | 63 ++ .../h2o_real_drop/__init__.py | 16 + .../h2o_real_drop/modify_llama.py | 784 ++++++++++++++++++ .../h2o_sim_drop/__init__.py | 16 + .../h2o_sim_drop/modify_gptneox.py | 284 +++++++ .../h2o_sim_drop/modify_llama.py | 218 +++++ .../h2o_sim_drop/modify_opt.py | 273 ++++++ 11 files changed, 2048 insertions(+) create mode 100644 examples/huggingface/pytorch/text-generation/h2o/generate_task_data.py create mode 100644 examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py create mode 100644 examples/huggingface/pytorch/text-generation/h2o/tasks/__init__.py create mode 100644 examples/huggingface/pytorch/text-generation/h2o/tasks/eval_harness.py create mode 100644 examples/huggingface/pytorch/text-generation/h2o/tasks/util.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py diff --git a/examples/huggingface/pytorch/text-generation/h2o/generate_task_data.py b/examples/huggingface/pytorch/text-generation/h2o/generate_task_data.py new file mode 100644 index 00000000000..2f44fc8d880 --- /dev/null +++ b/examples/huggingface/pytorch/text-generation/h2o/generate_task_data.py @@ -0,0 +1,74 @@ +import argparse +import json + +from lm_eval import evaluator, tasks +from tasks import EvalHarnessAdaptor + + +if __name__ == '__main__': + + + parser = argparse.ArgumentParser( + prog = 'ProgramName', + description = 'What the program does', + epilog = 'Text at the bottom of help') + + parser.add_argument('--output-file', type=str, default='input.jsonl') + parser.add_argument('--task-name', type=str, default='hellaswag') + parser.add_argument('--num-fewshot', type=int, default=0) + args = parser.parse_args() + + seq = 1024 + total_batch = 1 + pe = 'fixed' + + with open(args.output_file, 'w') as f: + pass + + class DryRunner: + def eval(self, batch): + + with open(args.output_file, 'a') as f: + for text in batch['text']: + item = { + "best_of": 1, + "echo": True, + "logprobs": 1, + "max_tokens": 0, + "model": "x", + "n": 1, + "prompt": text, + "request_type": "language-model-inference", + "stop": None, + "temperature": 0, + "top_p": 1 + } + f.write(json.dumps(item) + '\n') + + out = { + 'mask_loss': [1.0] * len(batch), + 'each_correct': [True] * len(batch), + } + return out + + t = DryRunner() + adaptor = EvalHarnessAdaptor(t, seq, total_batch, shrink=pe != "fixed") + results = evaluator.evaluate(adaptor, tasks.get_task_dict([args.task_name + #"lambada_openai", + #"piqa", + #"hellaswag", + #"winogrande", + #"mathqa", + #"pubmedqa", + # "boolq", + # "cb", + # "copa", + # "multirc", + # "record", + # "wic", + # "wsc", + ]), False, args.num_fewshot, None) + print('Finished') + + # dumped = json.dumps(results, indent=2) + # print(dumped) \ No newline at end of file diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py b/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py new file mode 100644 index 00000000000..f2ae8fbfdf2 --- /dev/null +++ b/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py @@ -0,0 +1,211 @@ +import argparse +import json, tqdm +import torch +import copy + +from lm_eval import evaluator, tasks +from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig + +from intel_extension_for_transformers.transformers.modeling.kv_cahe_compression.h2o_sim_drop.modify_llama import convert_kvcache_llama_heavy_recent +from intel_extension_for_transformers.transformers.modeling.kv_cahe_compression.h2o_sim_drop.modify_opt import convert_kvcache_opt_heavy_recent +from intel_extension_for_transformers.transformers.modeling.kv_cahe_compression.h2o_sim_drop.modify_gptneox import convert_kvcache_gpt_neox_heavy_recent + +from tasks import EvalHarnessAdaptor + +ENABLE_Heavy_Hitter_FUNCTIONS = { + "llama": convert_kvcache_llama_heavy_recent, + "opt": convert_kvcache_opt_heavy_recent, + "gpt_neox": convert_kvcache_gpt_neox_heavy_recent, +} + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + prog = 'ProgramName', + description = 'What the program does', + epilog = 'Text at the bottom of help') + parser.add_argument('--task_name', type=str, default='hellaswag') + parser.add_argument('--num_fewshot', type=int, default=0) + + parser.add_argument('--enable_small_cache', action='store_true') + parser.add_argument('--model_name', type=str, default='facebook/opt-350m') + parser.add_argument('--model_type', type=str, default='opt') + parser.add_argument("--cache_dir", type=str, default=None) + + parser.add_argument("--heavy_ratio", type=float, default=0.1) + parser.add_argument("--recent_ratio", type=float, default=0.1) + parser.add_argument("--device", type=str, default='cpu') + parser.add_argument("--seq_len", type=int, default=1024) + + parser.add_argument('--debug', action='store_true') + args = parser.parse_args() + + batch_size = 1 + pe = 'fixed' + seq = args.seq_len + + # build data + requests = [] + class DryRunner: + def eval(self, batch): + for text in batch['text']: + item = { + "best_of": 1, + "echo": True, + "logprobs": 1, + "max_tokens": 0, + "model": "x", + "n": 1, + "prompt": text, + "request_type": "language-model-inference", + "stop": None, + "temperature": 0, + "top_p": 1 + } + requests.append(item) + out = { + 'mask_loss': [1.0] * len(batch), + 'each_correct': [True] * len(batch), + } + return out + t = DryRunner() + adaptor = EvalHarnessAdaptor(t, seq, batch_size, shrink=pe != "fixed") + result = evaluator.evaluate(adaptor, tasks.get_task_dict([args.task_name]), False, args.num_fewshot, None) + + model_name = args.model_name + if 'cpu' in args.device: + device = args.device + else: + device = f"cuda:{args.device}" + + config = AutoConfig.from_pretrained(model_name, cache_dir=args.cache_dir) + tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=args.cache_dir) + model = AutoModelForCausalLM.from_pretrained(model_name, cache_dir=args.cache_dir) + + if args.enable_small_cache: + print('Enable Small Cache Size') + config.heavy_ratio = args.heavy_ratio + config.recent_ratio = args.recent_ratio + checkpoint = copy.deepcopy(model.state_dict()) + model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) + model.load_state_dict(checkpoint) + + model = model.to(device) + model.eval() + # model.half().eval() + + results = [] + with torch.no_grad(): + for request in tqdm.tqdm(requests): + result = {'request': request, 'result': {}} + prompt = request['prompt'] + input_ids = tokenizer(prompt, add_special_tokens=False, return_tensors='pt').input_ids.to(model.device) + + logits = model(input_ids).logits.log_softmax(dim=-1) + + values, indices = logits.squeeze(0).topk(dim=-1, k=1) + tokens = tokenizer.convert_ids_to_tokens(input_ids.squeeze(0)) + + gold_indices = input_ids[:, 1:] # skip first + logprobs = [None] + torch.gather(logits, -1, gold_indices.unsqueeze(-1)).squeeze(-1).squeeze(0).detach().cpu().tolist() + top_logprobs = [None] + [{tokenizer.convert_ids_to_tokens(i.item()): v.item()} for v, i in zip(values.squeeze(-1), indices.squeeze(-1))] + + result['result'] = { + "choices": [ + { + "text": prompt, + "logprobs": { + "tokens": tokens, + "token_logprobs": logprobs, + "top_logprobs": top_logprobs, + "text_offset": [] + }, + "finish_reason": "length" + } + ], + "request_time": { + "batch_time": 0, + "batch_size": 1} + } + + results.append(result) + + # evaluate + class RealRunner: + def __init__(self, args): + self.results = {} + for item in results: + request = item['request'] + result = item['result'] + self.results[json.dumps(request)] = result + print(f"{len(self.results)} items in the cache") + + def eval(self, batch): + from tasks.eval_harness import tokenizer + mask_loss = [] + each_correct = [] + for i, text in enumerate(batch['text']): + request = { + "best_of": 1, + "echo": True, + "logprobs": 1, + "max_tokens": 0, + "model": "x", + "n": 1, + "prompt": text, + "request_type": "language-model-inference", + "stop": None, + "temperature": 0, + "top_p": 1 + } + + key = json.dumps(request) + correct = True + + if key in self.results: + result = self.results[key] + token_logprobs = result['choices'][0]['logprobs']['token_logprobs'] + tokens = result['choices'][0]['logprobs']['tokens'] + top_logprobs = result['choices'][0]['logprobs']['top_logprobs'] + assert token_logprobs[0] is None + token_ids = tokenizer.convert_tokens_to_ids(tokens) + obs = batch['obs'][i] + target = batch['target'][i] + eval_mask = batch['eval_mask'][i] + + n_positive = 0 + sum_lobprob = 0 + if args.debug: + print(target) + for i, mask in enumerate(eval_mask): + try: + if i+1 >= len(tokens): + break + if mask == True: + if args.debug: + print(tokens[i+1], next(iter(top_logprobs[i+1].keys()))) + correct = correct and (tokens[i+1] == next(iter(top_logprobs[i+1].keys()))) + sum_lobprob += token_logprobs[i+1] + n_positive += 1 + except Exception as e: + raise e + # avg_logprob = sum(token_logprobs[1:]) / (len(token_logprobs) - 1) + avg_logprob = sum_lobprob / n_positive + mask_loss.append( - avg_logprob) + each_correct.append( correct ) + else: + assert False + + out = { + 'mask_loss': mask_loss, + 'each_correct': each_correct, + } + return out + + t = RealRunner(args) + + adaptor = EvalHarnessAdaptor(t, seq, batch_size, shrink=pe != "fixed") + results = evaluator.evaluate(adaptor, tasks.get_task_dict([args.task_name]), False, args.num_fewshot, None) + + dumped = json.dumps(results, indent=2) + print(dumped) \ No newline at end of file diff --git a/examples/huggingface/pytorch/text-generation/h2o/tasks/__init__.py b/examples/huggingface/pytorch/text-generation/h2o/tasks/__init__.py new file mode 100644 index 00000000000..f8ef348d72a --- /dev/null +++ b/examples/huggingface/pytorch/text-generation/h2o/tasks/__init__.py @@ -0,0 +1 @@ +from tasks.eval_harness import EvalHarnessAdaptor \ No newline at end of file diff --git a/examples/huggingface/pytorch/text-generation/h2o/tasks/eval_harness.py b/examples/huggingface/pytorch/text-generation/h2o/tasks/eval_harness.py new file mode 100644 index 00000000000..94528f13bec --- /dev/null +++ b/examples/huggingface/pytorch/text-generation/h2o/tasks/eval_harness.py @@ -0,0 +1,108 @@ +from functools import partial + +import os +import transformers +from lm_eval.base import LM +from tqdm import tqdm +import numpy as np + +from .util import sample_batch, shrink_seq +import multiprocessing +import ftfy + +from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig + +tokenizer = None + +def process_init(): + global tokenizer + model_name = os.environ.get('MODEL_NAME', 'facebook/opt-1.3b') + + if model_name == "EleutherAI/gpt-neox-20b": + tokenizer = AutoTokenizer.from_pretrained(model_name) + tokenizer.model_max_length = int(1e30) + tokenizer.pad_token = "<|endoftext|>" + elif model_name == 'huggyllama/llama-7b': + tokenizer = AutoTokenizer.from_pretrained(model_name) + tokenizer.model_max_length = int(1e30) + tokenizer.pad_token = "<|endoftext|>" + else: + tokenizer = AutoTokenizer.from_pretrained(model_name) + tokenizer.add_bos_token = False + +def process_request(x, seq): + global tokenizer + + ctx, cont = x +# ctx_tokens = tokenizer.encode("<|endoftext|>" + ftfy.fix_text(ctx, normalization="NFKC")) + ctx_text = ftfy.fix_text(ctx, normalization="NFKC") + cont_text = ftfy.fix_text(cont, normalization="NFKC") + all_text = ctx_text + cont_text + + ctx_tokens = tokenizer(ctx_text, add_special_tokens=False)['input_ids'] + cont_tokens = tokenizer(cont_text, add_special_tokens=False)['input_ids'] + + all_tokens = ctx_tokens + cont_tokens + all_tokens = np.array(all_tokens)[-seq:] # truncate sequence at seq length + + provided_ctx = len(all_tokens) - 1 + pad_amount = seq - provided_ctx + + return { + "obs": np.pad(all_tokens[:-1], ((0, pad_amount),), constant_values=tokenizer.pad_token_id), + "target": np.pad(all_tokens[1:], ((0, pad_amount),), constant_values=tokenizer.pad_token_id), + "ctx_length": seq, + "eval_mask": np.logical_and( + np.arange(0, seq) > len(all_tokens) - len(cont_tokens) - 2, + np.arange(0, seq) < len(all_tokens) - 1 + ), + "prompt": ctx_text, + "target": cont_text, + "text": all_text, + } + + +class EvalHarnessAdaptor(LM): + def greedy_until(self, requests): + raise Exception("unimplemented") + + def loglikelihood_rolling(self, requests): + raise Exception("unimplemented") + + def __init__(self, tpu_cluster, seq, batch, shrink, min_seq=None): + super().__init__() + self.tpu = tpu_cluster + self.seq = seq + self.batch = batch + self.shrink = shrink + self.min_seq = min_seq + + self.pool = multiprocessing.Pool(processes=1, initializer=process_init) + # self.pool = multiprocessing.Pool(initializer=process_init) + process_init() + + def convert_requests(self, requests): + return self.pool.imap(partial(process_request, seq=self.seq), requests) + + def loglikelihood(self, requests): + output = [] + + r = self.convert_requests(requests) + self.pool.close() + zero_example = process_request(requests[0], self.seq) + + for b in tqdm(sample_batch(r, self.batch, zero_example), + desc="LM eval harness", + total=len(requests) // self.batch): + + if self.shrink: + b = shrink_seq(b, min_seq=self.min_seq) + + out = self.tpu.eval(b) + + for loss, correct in zip(out["mask_loss"], out["each_correct"]): + output.append((float(-loss), bool(correct))) + + return output + + diff --git a/examples/huggingface/pytorch/text-generation/h2o/tasks/util.py b/examples/huggingface/pytorch/text-generation/h2o/tasks/util.py new file mode 100644 index 00000000000..6cf9080d3d8 --- /dev/null +++ b/examples/huggingface/pytorch/text-generation/h2o/tasks/util.py @@ -0,0 +1,63 @@ +from itertools import zip_longest + +import numpy as np + + +def grouper(n, iterable, fillvalue): + "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" + args = [iter(iterable)] * n + return zip_longest(fillvalue=fillvalue, *args) + + +# divide the seq length by 2 until it would truncate actual context +def shrink_seq(examples, min_seq=None): + length = examples["obs"].shape[-1] + + new_length = length // 2 + + if min_seq is not None: + if new_length < min_seq: + return examples + + max_length = np.max(examples["eval_mask"] * np.arange(0, length)) + 1 + + if max_length < new_length: + examples["obs"] = examples["obs"][:, :new_length] + examples["target"] = examples["target"][:, :new_length] + examples["eval_mask"] = examples["eval_mask"][:, :new_length] + + return shrink_seq(examples, min_seq=min_seq) + else: + return examples + + +def sample_batch(examples, bs, zero_example_shape): + zero_example = { + "obs": np.zeros_like(zero_example_shape["obs"]), + "target": np.zeros_like(zero_example_shape["target"]), + "eval_mask": np.zeros_like(zero_example_shape["eval_mask"]), + "ctx_length": 0, + } + + for batch in grouper(bs, examples, zero_example): + batch_flattened = { + "obs": [], + "target": [], + "eval_mask": [], + "ctx_length": [], + "text": [], + } + + for sample in batch: + batch_flattened["obs"].append(sample["obs"]) + batch_flattened["target"].append(sample["target"]) + batch_flattened["eval_mask"].append(sample["eval_mask"]) + batch_flattened["ctx_length"].append(sample["ctx_length"]) + batch_flattened["text"].append(sample["text"]) + + batch_flattened["obs"] = np.array(batch_flattened["obs"]) + batch_flattened["target"] = np.array(batch_flattened["target"]) + batch_flattened["eval_mask"] = np.array(batch_flattened["eval_mask"]) + batch_flattened["ctx_length"] = np.array(batch_flattened["ctx_length"]) + + yield batch_flattened \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py new file mode 100644 index 00000000000..db2d9048f2d --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py new file mode 100644 index 00000000000..247cb4d25fd --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py @@ -0,0 +1,784 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from typing import Optional, Tuple + +import pdb +import torch +from torch import nn +import torch.utils.checkpoint + +import torch.nn.functional as F + +# from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import ( + LlamaAttention, + rotate_half, + apply_rotary_pos_emb, + LlamaRotaryEmbedding, + apply_rotary_pos_emb, + LlamaForCausalLM, + LlamaLinearScalingRotaryEmbedding, + LlamaDynamicNTKScalingRotaryEmbedding +) +import types + +__all__ = ["H2OLlamaForCausalLM", "H2OLlamaAttention", + 'H2OLlamaAttention_streaming', 'H2OLlamaForCausalLM_streaming'] + + +from transformers.configuration_utils import PretrainedConfig + +LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP = {} + +class LlamaConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the LLaMA-7B. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`LlamaModel`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 11008): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`int`, *optional*): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 2048): + The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens, + Llama 2 up to 4096, CodeLlama up to 16384. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-06): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + pad_token_id (`int`, *optional*): + Padding token id. + bos_token_id (`int`, *optional*, defaults to 1): + Beginning of stream token id. + eos_token_id (`int`, *optional*, defaults to 2): + End of stream token id. + pretraining_tp (`int`, *optional*, defaults to 1): + Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this + document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is + necessary to ensure exact reproducibility of the pretraining results. Please refer to [this + issue](https://github.com/pytorch/pytorch/issues/76232). + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`Dict`, *optional*): + Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling + strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is + `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update + `max_position_embeddings` to the expected new maximum. See the following thread for more information on how + these scaling strategies behave: + https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an + experimental feature, subject to breaking API changes in future versions. + attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`): + Whether to use a bias in the query, key, value and output projection layers during self-attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + + ```python + >>> from transformers import LlamaModel, LlamaConfig + + >>> # Initializing a LLaMA llama-7b style configuration + >>> configuration = LlamaConfig() + + >>> # Initializing a model from the llama-7b style configuration + >>> model = LlamaModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "llama" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=32000, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self._rope_scaling_validation() + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + def _rope_scaling_validation(self): + """ + Validate the `rope_scaling` configuration. + """ + if self.rope_scaling is None: + return + + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2: + raise ValueError( + "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, " + f"got {self.rope_scaling}" + ) + rope_scaling_type = self.rope_scaling.get("type", None) + rope_scaling_factor = self.rope_scaling.get("factor", None) + if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]: + raise ValueError( + f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}" + ) + if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0: + raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}") + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + +def _make_causal_mask( + bsz: int, tgt_len: int, past_key_values_length: int, dtype: torch.dtype, device: torch.device): + """ + Make causal mask used for bi-directional self-attention. + """ + mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +def apply_rotary_pos_emb_single(x, cos, sin, position_ids): + # The first two dimensions of cos and sin are always 1, so we can `squeeze` them. + cos = cos.squeeze(1).squeeze(0) # [seq_len, dim] + sin = sin.squeeze(1).squeeze(0) # [seq_len, dim] + cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + x_embed = (x * cos) + (rotate_half(x) * sin) + return x_embed + + +class H2OKVCache_LayerWise: + def __init__( + self, + hh_size=4, + recent_size=512, + k_seq_dim=2, + v_seq_dim=2, + ): + print(f"H2OKVCache-LayerWise: {hh_size}, {recent_size}") + self.hh_size = hh_size + self.recent_size = recent_size + self.cache_size = hh_size + recent_size + self.k_seq_dim = k_seq_dim + self.v_seq_dim = v_seq_dim + self.hh_score = None + + def __call__(self, past_key_values, attn_score_cache): + + self._update_hh_score(attn_score_cache) + + if past_key_values is None: + return None + seq_len = past_key_values[0].size(self.k_seq_dim) + if seq_len <= self.cache_size: + return past_key_values + + # hh-selection + bsz, num_heads, _, head_dim = past_key_values[0].shape + + select_hh_scores = self.hh_score[:, :seq_len - self.recent_size] + _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) + keep_topk = keep_topk.sort().values + + # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) + keep_recent = torch.arange(seq_len - self.recent_size, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) + keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) + + mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) + mask = mask.scatter(-1, keep_idx, 1) + + k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + + self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) + + return (k_hh_recent, v_hh_recent) + + def evict_for_space(self, past_key_values, num_coming): + if past_key_values is None: + return None + seq_len = past_key_values[0][0].size(self.k_seq_dim) + if seq_len + num_coming <= self.cache_size: + return past_key_values + + # hh-selection + bsz, num_heads, _, head_dim = past_key_values[0].shape + + select_hh_scores = self.hh_score[:, :seq_len - self.recent_size + num_coming] + _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) + keep_topk = keep_topk.sort().values + + # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) + keep_recent = torch.arange(seq_len - self.recent_size + num_coming, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) + keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) + + mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) + mask = mask.scatter(-1, keep_idx, 1) + + k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + + self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) + + return (k_hh_recent, v_hh_recent) + + def _update_hh_score(self, attn_score_cache): + + num_new_tokens = attn_score_cache.shape[2] + + if self.hh_score is None: + self.hh_score = attn_score_cache.sum(0).sum(1) + else: + attn_score_cache = attn_score_cache.sum(0).sum(1) + attn_score_cache[:, :-num_new_tokens] += self.hh_score + self.hh_score = attn_score_cache + + def _clean_scores(self): + self.hh_score = None + + +class H2OLlamaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: LlamaConfig): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self._init_rope() + + self.kv_cache = H2OKVCache_LayerWise( + hh_size=config.hh_size, + recent_size=config.recent_size, + k_seq_dim=2, + v_seq_dim=2, + ) + + def _init_rope(self): + if self.config.rope_scaling is None: + self.rotary_emb = LlamaRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + else: + scaling_type = self.config.rope_scaling["type"] + scaling_factor = self.config.rope_scaling["factor"] + if scaling_type == "linear": + self.rotary_emb = LlamaLinearScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + scaling_factor=scaling_factor, + base=self.rope_theta, + ) + elif scaling_type == "dynamic": + self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + scaling_factor=scaling_factor, + base=self.rope_theta, + ) + else: + raise ValueError(f"Unknown RoPE scaling type {scaling_type}") + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def _clean_cache(self): + self.kv_cache._clean_scores() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + + bsz, q_len, _ = hidden_states.size() + + if self.config.pretraining_tp > 1: + key_value_slicing = ( + self.num_key_value_heads * self.head_dim + ) // self.config.pretraining_tp + query_slices = self.q_proj.weight.split( + (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 + ) + key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) + value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) + + query_states = [ + F.linear(hidden_states, query_slices[i]) + for i in range(self.config.pretraining_tp) + ] + query_states = torch.cat(query_states, dim=-1) + + key_states = [ + F.linear(hidden_states, key_slices[i]) + for i in range(self.config.pretraining_tp) + ] + key_states = torch.cat(key_states, dim=-1) + + value_states = [ + F.linear(hidden_states, value_slices[i]) + for i in range(self.config.pretraining_tp) + ] + value_states = torch.cat(value_states, dim=-1) + + else: + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view( + bsz, q_len, self.num_heads, self.head_dim + ).transpose(1, 2) + key_states = key_states.view( + bsz, q_len, self.num_key_value_heads, self.head_dim + ).transpose(1, 2) + value_states = value_states.view( + bsz, q_len, self.num_key_value_heads, self.head_dim + ).transpose(1, 2) + + # remake causal mask + attention_mask = _make_causal_mask( + bsz=bsz, + tgt_len=q_len, + past_key_values_length=past_key_value[0].shape[-2] if past_key_value is not None else 0, + dtype=query_states.dtype, + device=query_states.device, + ) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + + position_length = kv_seq_len + if not position_ids.nelement() > 1: + if position_length < position_ids.item()+1: + position_length = position_ids.item()+1 + + cos, sin = self.rotary_emb(value_states, seq_len=position_length) + ### Shift Pos: query pos is min(cache_size, idx) + # query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + query_states = apply_rotary_pos_emb_single(query_states, cos, sin, position_ids) + key_states = apply_rotary_pos_emb_single(key_states, cos, sin, position_ids) + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt( + self.head_dim + ) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to( + query_states.dtype + ) + + past_key_value = self.kv_cache(past_key_value, attn_weights.detach().clone()) + + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + if self.config.pretraining_tp > 1: + attn_output = attn_output.split( + self.hidden_size // self.config.pretraining_tp, dim=2 + ) + o_proj_slices = self.o_proj.weight.split( + self.hidden_size // self.config.pretraining_tp, dim=1 + ) + attn_output = sum( + [ + F.linear(attn_output[i], o_proj_slices[i]) + for i in range(self.config.pretraining_tp) + ] + ) + else: + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class H2OLlamaForCausalLM(LlamaForCausalLM): + def __init__(self, config): + super().__init__(config) + num_layers = len(self.model.layers) + for layer_idx in range(num_layers): + self.model.layers[layer_idx].self_attn = H2OLlamaAttention(config) + + + +## H2O KV Cache dropping with Position rolling +class H2OLlamaAttention_streaming(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: LlamaConfig): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self._init_rope() + + self.kv_cache = H2OKVCache_LayerWise( + hh_size=config.hh_size, + recent_size=config.recent_size, + k_seq_dim=2, + v_seq_dim=2, + ) + + def _init_rope(self): + if self.config.rope_scaling is None: + self.rotary_emb = LlamaRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + else: + scaling_type = self.config.rope_scaling["type"] + scaling_factor = self.config.rope_scaling["factor"] + if scaling_type == "linear": + self.rotary_emb = LlamaLinearScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + scaling_factor=scaling_factor, + base=self.rope_theta, + ) + elif scaling_type == "dynamic": + self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + scaling_factor=scaling_factor, + base=self.rope_theta, + ) + else: + raise ValueError(f"Unknown RoPE scaling type {scaling_type}") + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def _clean_cache(self): + self.kv_cache._clean_scores() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + + bsz, q_len, _ = hidden_states.size() + + if self.config.pretraining_tp > 1: + key_value_slicing = ( + self.num_key_value_heads * self.head_dim + ) // self.config.pretraining_tp + query_slices = self.q_proj.weight.split( + (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 + ) + key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) + value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) + + query_states = [ + F.linear(hidden_states, query_slices[i]) + for i in range(self.config.pretraining_tp) + ] + query_states = torch.cat(query_states, dim=-1) + + key_states = [ + F.linear(hidden_states, key_slices[i]) + for i in range(self.config.pretraining_tp) + ] + key_states = torch.cat(key_states, dim=-1) + + value_states = [ + F.linear(hidden_states, value_slices[i]) + for i in range(self.config.pretraining_tp) + ] + value_states = torch.cat(value_states, dim=-1) + + else: + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view( + bsz, q_len, self.num_heads, self.head_dim + ).transpose(1, 2) + key_states = key_states.view( + bsz, q_len, self.num_key_value_heads, self.head_dim + ).transpose(1, 2) + value_states = value_states.view( + bsz, q_len, self.num_key_value_heads, self.head_dim + ).transpose(1, 2) + + # remake causal mask + attention_mask = _make_causal_mask( + bsz=bsz, + tgt_len=q_len, + past_key_values_length=past_key_value[0].shape[-2] if past_key_value is not None else 0, + dtype=query_states.dtype, + device=query_states.device, + ) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + + if not position_ids.nelement() > 1: + position_ids[0][0] = kv_seq_len - 1 + + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + ### Shift Pos: query pos is min(cache_size, idx) + # query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + query_states = apply_rotary_pos_emb_single(query_states, cos, sin, position_ids) + ### + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + ### Shift Pos: key pos is the pos in cache (Rolling KV Cache and using relative pos emb) + key_position_ids = torch.arange(kv_seq_len, device=position_ids.device).unsqueeze(0) + key_states = apply_rotary_pos_emb_single(key_states, cos, sin, key_position_ids) + ### + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt( + self.head_dim + ) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to( + query_states.dtype + ) + + past_key_value = self.kv_cache(past_key_value, attn_weights.detach().clone()) + + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + if self.config.pretraining_tp > 1: + attn_output = attn_output.split( + self.hidden_size // self.config.pretraining_tp, dim=2 + ) + o_proj_slices = self.o_proj.weight.split( + self.hidden_size // self.config.pretraining_tp, dim=1 + ) + attn_output = sum( + [ + F.linear(attn_output[i], o_proj_slices[i]) + for i in range(self.config.pretraining_tp) + ] + ) + else: + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class H2OLlamaForCausalLM_streaming(LlamaForCausalLM): + def __init__(self, config): + super().__init__(config) + num_layers = len(self.model.layers) + for layer_idx in range(num_layers): + self.model.layers[layer_idx].self_attn = H2OLlamaAttention_streaming(config) + + + + + diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py new file mode 100644 index 00000000000..db2d9048f2d --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py new file mode 100644 index 00000000000..01bc7b63e87 --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pdb +import copy +import math +import numpy as np +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch +from torch import nn +import torch.utils.checkpoint +import torch.nn.functional as F +from torch.cuda.amp import autocast +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from transformers.models.gpt_neox.modeling_gpt_neox import GPTNeoXRotaryEmbedding, GPTNeoXAttention, apply_rotary_pos_emb +# from transformers.models.gpt_neox.modeling_gpt_neox import GPTNeoXRotaryEmbedding + + +__all__ = ['convert_kvcache_gpt_neox_heavy_recent', 'GPTNeoXAttention_Mask'] + +def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): + + # attn_weights (BS, head, query, keys) + dtype_attn_weights = attn_weights.dtype + seq_length = attn_weights.shape[-1] + if no_padding_seq_length is None: + padding_length = 0 + else: + padding_length = seq_length - no_padding_seq_length + + offset = torch.finfo(attn_weights.dtype).min + tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) + + accumulated_attention_score = torch.sum(tmp_attn[:,:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) + accumulated_attention_score[:,:,heavy_budget+padding_length:] = 0 + if padding_length > 0: + accumulated_attention_score[:,:,:padding_length] = 0 + + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + mask_bottom[:,:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True + + for token_index in range(heavy_budget+padding_length, seq_length): + + tmp_attn_index = nn.functional.softmax(attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) + zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) + mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) + mask_bottom_index[:,:, token_index] = True + + mask_bottom[:,:,token_index,:] = mask_bottom_index + accumulated_attention_score += tmp_attn_index + accumulated_attention_score = accumulated_attention_score * mask_bottom_index + + return mask_bottom + +def sanity_check(mask): + # mask (head, query, key) + ones = torch.ones_like(mask) + ones = torch.triu(ones, diagonal=0) + mask_bottom = torch.logical_or(mask, ones) + + error_cnt = 0 + for i in range(mask_bottom.shape[1]-1): + index = mask_bottom[:,i,:].eq(0).unsqueeze(1) + index[:,i:]=0 + error_cnt += (mask_bottom[:,i:,:] * index).sum().item() + print(error_cnt) + return error_cnt + +class GPTNeoXAttention_Mask(nn.Module): + def __init__(self, config): + super().__init__() + self.num_attention_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.num_attention_heads + self.rotary_ndims = int(self.head_size * config.rotary_pct) + max_positions = config.max_position_embeddings + self.register_buffer( + "bias", + torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view( + 1, 1, max_positions, max_positions + ), + ) + self.register_buffer("masked_bias", torch.tensor(-1e9)) + # self.rotary_emb = RotaryEmbedding( + self.rotary_emb = GPTNeoXRotaryEmbedding( + self.rotary_ndims, config.max_position_embeddings, base=config.rotary_emb_base + ) + self.register_buffer( + "norm_factor", + torch.sqrt(torch.tensor(self.head_size, dtype=torch.float32)).to(torch.get_default_dtype()), + persistent=False, + ) + self.query_key_value = nn.Linear(config.hidden_size, 3 * config.hidden_size) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + + self.heavy_budget_ratio = config.heavy_ratio + self.recent_budget_ratio = config.recent_ratio + + def forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: torch.FloatTensor, + position_ids: torch.LongTensor, + head_mask: Optional[torch.FloatTensor] = None, + layer_past: Optional[Tuple[torch.Tensor]] = None, + use_cache: Optional[bool] = False, + output_attentions: Optional[bool] = False, + ): + has_layer_past = layer_past is not None + + # Compute QKV + # Attention heads [batch, seq_len, hidden_size] + # --> [batch, seq_len, (np * 3 * head_size)] + qkv = self.query_key_value(hidden_states) + + # [batch, seq_len, (num_heads * 3 * head_size)] + # --> [batch, seq_len, num_heads, 3 * head_size] + new_qkv_shape = qkv.size()[:-1] + (self.num_attention_heads, 3 * self.head_size) + qkv = qkv.view(*new_qkv_shape) + + # [batch, seq_len, num_attention_heads, 3 * head_size] --> 3 [batch, num_attention_heads, seq_len, head_size] + query = qkv[..., : self.head_size].permute(0, 2, 1, 3) + key = qkv[..., self.head_size : 2 * self.head_size].permute(0, 2, 1, 3) + value = qkv[..., 2 * self.head_size :].permute(0, 2, 1, 3) + + # Compute rotary embeddings on rotary_ndims + query_rot = query[..., : self.rotary_ndims] + query_pass = query[..., self.rotary_ndims :] + key_rot = key[..., : self.rotary_ndims] + key_pass = key[..., self.rotary_ndims :] + + # Compute token offset for rotary embeddings (when decoding) + seq_len = key.shape[-2] + if has_layer_past: + seq_len += layer_past[0].shape[-2] + cos, sin = self.rotary_emb(value, seq_len=seq_len) + query, key = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids) + query = torch.cat((query, query_pass), dim=-1) + key = torch.cat((key, key_pass), dim=-1) + + # Cache QKV values + if has_layer_past: + past_key = layer_past[0] + past_value = layer_past[1] + key = torch.cat((past_key, key), dim=-2) + value = torch.cat((past_value, value), dim=-2) + present = (key, value) if use_cache else None + + # Compute attention + attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask) + + # Reshape outputs + attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_size) + attn_output = self.dense(attn_output) + + outputs = (attn_output, present) + if output_attentions: + outputs += (attn_weights,) + + return outputs + + @classmethod + def _split_heads(cls, tensor, num_attention_heads, attn_head_size): + """ + Splits hidden dim into attn_head_size and num_attention_heads + """ + # tensor: [bs, seq_len, hidden_size] + new_shape = tensor.size()[:-1] + (num_attention_heads, attn_head_size) + # -> [bs, seq_len, num_attention_heads, attn_head_size] + tensor = tensor.view(new_shape) + # -> [bs, num_attention_heads, seq_len, attn_head_size] + tensor = tensor.permute(0, 2, 1, 3) + return tensor + + @classmethod + def _merge_heads(cls, tensor, num_attention_heads, attn_head_size): + """ + Merges attn_head_size dim and num_attn_heads dim into hidden dim + """ + # tensor [bs, num_attention_heads, seq_len, attn_head_size] + tensor = tensor.permute(0, 2, 1, 3).contiguous() + # -> [bs, seq_len, num_attention_heads, attn_head_size] + tensor = tensor.view(tensor.size(0), tensor.size(1), num_attention_heads * attn_head_size) + # -> [bs, seq_len, hidden_size] + return tensor + + def _attn(self, query, key, value, attention_mask=None, head_mask=None): + # q, k, v: [bs, num_attention_heads, seq_len, attn_head_size] + # compute causal mask from causal mask buffer + batch_size, num_attention_heads, query_length, attn_head_size = query.size() + key_length = key.size(-2) + + causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length] + + query = query.view(batch_size * num_attention_heads, query_length, attn_head_size) + key = key.view(batch_size * num_attention_heads, key_length, attn_head_size) + attn_scores = torch.zeros( + batch_size * num_attention_heads, + query_length, + key_length, + dtype=query.dtype, + device=key.device, + ) + attn_scores = torch.baddbmm( + attn_scores, + query, + key.transpose(1, 2), + beta=1.0, + alpha=(torch.tensor(1.0, dtype=self.norm_factor.dtype, device=self.norm_factor.device) / self.norm_factor), + ) + attn_scores = attn_scores.view(batch_size, num_attention_heads, query_length, key_length) + + mask_value = torch.finfo(attn_scores.dtype).min + # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`. + # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device` + mask_value = torch.tensor(mask_value, dtype=attn_scores.dtype).to(attn_scores.device) + attn_scores = torch.where(causal_mask, attn_scores, mask_value) + + if attention_mask is not None: + # Apply the attention mask + attn_scores = attn_scores + attention_mask + + #attn_scores (bs, head, token, token) + ### Heavy + Recent + heavy_budget = int(self.heavy_budget_ratio * attn_scores.shape[-1]) + recent_budget = int(self.recent_budget_ratio * attn_scores.shape[-1]) + + # Heavy Hitter Mask + if heavy_budget > 0: + mask_bottom = local_heavy_hitter_mask(attn_scores, heavy_budget, None) # Default: No padding applied to input + else: + mask_bottom = torch.zeros_like(attn_scores, dtype=torch.bool) + + ones = torch.ones_like(attn_scores, dtype=torch.bool) + ones = torch.triu(ones, diagonal=-recent_budget) + mask_bottom = torch.logical_or(mask_bottom, ones) + + mask_bottom = torch.tril(mask_bottom, diagonal=0) + + # mask_bottom = ones + attn_scores[~mask_bottom] = torch.finfo(attn_scores.dtype).min + attn_weights = nn.functional.softmax(attn_scores, dim=-1) + + attn_weights = attn_weights.to(value.dtype) + + # Mask heads if we want to + if head_mask is not None: + attn_weights = attn_weights * head_mask + + attn_output = torch.matmul(attn_weights, value) + return attn_output, attn_weights + +def convert_kvcache_gpt_neox_heavy_recent(model, config): + + for name, module in reversed(model._modules.items()): + + if len(list(module.children())) > 0: + model._modules[name] = convert_kvcache_gpt_neox_heavy_recent(module, config) + + if isinstance(module, GPTNeoXAttention): + model._modules[name] = GPTNeoXAttention_Mask(config) + + return model + + diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py new file mode 100644 index 00000000000..4ca213fa0a1 --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pdb +import copy +import math +import numpy as np +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch +from torch import nn +import torch.utils.checkpoint +import torch.nn.functional as F +from torch.cuda.amp import autocast +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + + +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding, LlamaAttention, apply_rotary_pos_emb + + +__all__ = ['convert_kvcache_llama_heavy_recent', 'LlamaAttention_heavy_hitter'] + + +def local_heavy_hitter_mask(attn_weights, heavy_budget): + + # attn_weights (BS, head, query, keys) + dtype_attn_weights = attn_weights.dtype + seq_length = attn_weights.shape[-1] + padding_length = 0 + + offset = torch.finfo(attn_weights.dtype).min + tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) + + accumulated_attention_score = torch.sum(tmp_attn[:,:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) + accumulated_attention_score[:,:,heavy_budget+padding_length:] = 0 + accumulated_attention_score[:,:,:padding_length] = 0 + + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + mask_bottom[:,:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True + + for token_index in range(heavy_budget+padding_length, seq_length): + + tmp_attn_index = nn.functional.softmax(attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) + zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) + mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) + mask_bottom_index[:,:, token_index] = True + + mask_bottom[:,:,token_index,:] = mask_bottom_index + accumulated_attention_score += tmp_attn_index + accumulated_attention_score = accumulated_attention_score * mask_bottom_index + + return mask_bottom + + +class LlamaAttention_heavy_hitter(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: LlamaConfig): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.max_position_embeddings = config.max_position_embeddings + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.rotary_emb = LlamaRotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings) + + self.heavy_budget_ratio = config.heavy_ratio + self.recent_budget_ratio = config.recent_ratio + + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + # [bsz, nh, t, hd] + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + attn_weights = torch.max(attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)) + + ### Heavy + Recent + heavy_budget = int(self.heavy_budget_ratio * attn_weights.shape[-1]) + recent_budget = int(self.recent_budget_ratio * attn_weights.shape[-1]) + + + # # Heavy Hitter Mask (Based on local statistics) + # if heavy_budget > 0: + # mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget) # Default: No padding applied to input + # else: + # mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + + # ones = torch.ones_like(attn_weights, dtype=torch.bool) + # ones = torch.triu(ones, diagonal=-recent_budget) + # mask_bottom = torch.logical_or(mask_bottom, ones) + + # mask_bottom = torch.tril(mask_bottom, diagonal=0) + + # # mask_bottom = ones + # attn_weights[~mask_bottom] = torch.min(attention_mask) + + + + # Heavy Hitter Mask (Based on global statistics) + tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(attn_weights.dtype) + tmp_sum = torch.sum(tmp_attn, dim=-2) + _, tmp_topk = tmp_sum.topk(k=heavy_budget, dim=-1) + + zeros = torch.zeros_like(tmp_sum, dtype=torch.bool) + mask_bottom = zeros.scatter(-1, tmp_topk, True).unsqueeze(2) + mask_bottom = mask_bottom.expand(mask_bottom.shape[0], mask_bottom.shape[1], attn_weights.shape[-2], mask_bottom.shape[-1]) + + ones = torch.ones_like(attn_weights, dtype=torch.bool) + ones = torch.tril(ones, diagonal=recent_budget) + ones = torch.triu(ones, diagonal=-recent_budget) + mask_bottom = torch.logical_or(mask_bottom, ones) + # mask_bottom = ones + attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +def convert_kvcache_llama_heavy_recent(model, config): + + for name, module in reversed(model._modules.items()): + + if len(list(module.children())) > 0: + model._modules[name] = convert_kvcache_llama_heavy_recent(module, config) + + if isinstance(module, LlamaAttention): + model._modules[name] = LlamaAttention_heavy_hitter(config) + + return model + diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py new file mode 100644 index 00000000000..23cd926a3eb --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py @@ -0,0 +1,273 @@ +import os +import pdb +import copy +import math +import numpy as np +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch +from torch import nn +import torch.utils.checkpoint +import torch.nn.functional as F +from torch.cuda.amp import autocast +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from transformers.models.opt.modeling_opt import OPTAttention + + +__all__ = ['convert_kvcache_opt_heavy_recent', 'OPTAttention_Mask'] + + + +def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): + + # attn_weights (head, query, keys) + dtype_attn_weights = attn_weights.dtype + seq_length = attn_weights.shape[-1] + if no_padding_seq_length is None: + padding_length = 0 + else: + padding_length = seq_length - no_padding_seq_length + + offset = torch.finfo(attn_weights.dtype).min + tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) + + accumulated_attention_score = torch.sum(tmp_attn[:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) + accumulated_attention_score[:,heavy_budget+padding_length:] = 0 + + if padding_length > 0: + accumulated_attention_score[:,:padding_length] = 0 + + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + mask_bottom[:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True + + for token_index in range(heavy_budget+padding_length, seq_length): + + tmp_attn_index = nn.functional.softmax(attn_weights[:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) + zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) + mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) + mask_bottom_index[:, token_index] = True + + mask_bottom[:,token_index,:] = mask_bottom_index + accumulated_attention_score += tmp_attn_index + accumulated_attention_score = accumulated_attention_score * mask_bottom_index + + mask_bottom = torch.tril(mask_bottom, diagonal=0) + + return mask_bottom + + +def sanity_check(mask): + # mask (head, query, key) + ones = torch.ones_like(mask) + ones = torch.triu(ones, diagonal=0) + mask_bottom = torch.logical_or(mask, ones) + + error_cnt = 0 + for i in range(mask_bottom.shape[1]-1): + index = mask_bottom[:,i,:].eq(0).unsqueeze(1) + index[:,i:]=0 + error_cnt += (mask_bottom[:,i:,:] * index).sum().item() + print(error_cnt) + return error_cnt + + +class OPTAttention_Mask(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + embed_dim: int, + num_heads: int, + heavy_ratio: float, + recent_ratio: float, + dropout: float = 0.0, + is_decoder: bool = False, + bias: bool = True, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.is_decoder = is_decoder + + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + + self.heavy_budget_ratio = heavy_ratio + self.recent_budget_ratio = recent_ratio + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = torch.max(attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + ### Heavy + Recent + heavy_budget = int(self.heavy_budget_ratio * attn_weights.shape[-1]) + recent_budget = int(self.recent_budget_ratio * attn_weights.shape[-1]) + + # Heavy Hitter Mask + if heavy_budget > 0: + mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) # Default: No padding applied to input + else: + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + + # Recent Mask + ones = torch.ones_like(attn_weights, dtype=torch.bool) + ones = torch.triu(ones, diagonal=-recent_budget) + mask_bottom = torch.logical_or(mask_bottom, ones) + + # Combine h2o+recent and apply casual mask + mask_bottom = torch.tril(mask_bottom, diagonal=0) + # mask_bottom = ones + attn_weights[~mask_bottom] = torch.min(attention_mask) + + + # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 + if attn_weights.dtype == torch.float16: + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(torch.float16) + else: + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if layer_head_mask is not None: + if layer_head_mask.size() != (self.num_heads,): + raise ValueError( + f"Head mask for a single layer should be of size {(self.num_heads,)}, but is" + f" {layer_head_mask.size()}" + ) + attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if output_attentions: + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to be reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + + # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # partitioned aross GPUs when using tensor-parallelism. + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped, past_key_value + + +def convert_kvcache_opt_heavy_recent(model, config): + + for name, module in reversed(model._modules.items()): + + if len(list(module.children())) > 0: + model._modules[name] = convert_kvcache_opt_heavy_recent(module, config) + + if isinstance(module, OPTAttention): + model._modules[name] = OPTAttention_Mask( + embed_dim=module.embed_dim, + num_heads=config.num_attention_heads, + heavy_ratio = config.heavy_ratio, + recent_ratio = config.recent_ratio, + dropout=config.attention_dropout, + is_decoder=True, + bias=config.enable_bias, + ) + return model + From eb7f564478f5f3b9aa9e9ace0714bad6572022b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 07:37:50 +0000 Subject: [PATCH 02/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../h2o_real_drop/__init__.py | 2 +- .../h2o_real_drop/modify_llama.py | 28 +++++++------------ .../h2o_sim_drop/__init__.py | 2 +- .../h2o_sim_drop/modify_gptneox.py | 12 ++------ .../h2o_sim_drop/modify_llama.py | 11 ++++---- .../h2o_sim_drop/modify_opt.py | 19 +++++++++++-- 6 files changed, 36 insertions(+), 38 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py index db2d9048f2d..1e8078e9b40 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py @@ -13,4 +13,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py index 247cb4d25fd..2c97a930cbd 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py @@ -46,8 +46,7 @@ LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP = {} class LlamaConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA + r"""This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the LLaMA-7B. @@ -126,7 +125,8 @@ class LlamaConfig(PretrainedConfig): >>> # Accessing the model configuration >>> configuration = model.config - ```""" + ``` + """ model_type = "llama" keys_to_ignore_at_inference = ["past_key_values"] @@ -187,9 +187,7 @@ def __init__( ) def _rope_scaling_validation(self): - """ - Validate the `rope_scaling` configuration. - """ + """Validate the `rope_scaling` configuration.""" if self.rope_scaling is None: return @@ -209,8 +207,9 @@ def _rope_scaling_validation(self): def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + """This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). + + The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) """ batch, num_key_value_heads, slen, head_dim = hidden_states.shape @@ -221,9 +220,7 @@ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: def _make_causal_mask( bsz: int, tgt_len: int, past_key_values_length: int, dtype: torch.dtype, device: torch.device): - """ - Make causal mask used for bi-directional self-attention. - """ + """Make causal mask used for bi-directional self-attention.""" mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) @@ -335,7 +332,7 @@ def _clean_scores(self): class H2OLlamaAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" + """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__(self, config: LlamaConfig): super().__init__() @@ -557,7 +554,7 @@ def __init__(self, config): ## H2O KV Cache dropping with Position rolling class H2OLlamaAttention_streaming(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" + """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__(self, config: LlamaConfig): super().__init__() @@ -777,8 +774,3 @@ def __init__(self, config): num_layers = len(self.model.layers) for layer_idx in range(num_layers): self.model.layers[layer_idx].self_attn = H2OLlamaAttention_streaming(config) - - - - - diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py index db2d9048f2d..1e8078e9b40 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py @@ -13,4 +13,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py index 01bc7b63e87..72083f2c1c7 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py @@ -19,7 +19,7 @@ import pdb import copy import math -import numpy as np +import numpy as np from dataclasses import dataclass from typing import Optional, Tuple, Union @@ -180,9 +180,7 @@ def forward( @classmethod def _split_heads(cls, tensor, num_attention_heads, attn_head_size): - """ - Splits hidden dim into attn_head_size and num_attention_heads - """ + """Splits hidden dim into attn_head_size and num_attention_heads.""" # tensor: [bs, seq_len, hidden_size] new_shape = tensor.size()[:-1] + (num_attention_heads, attn_head_size) # -> [bs, seq_len, num_attention_heads, attn_head_size] @@ -193,9 +191,7 @@ def _split_heads(cls, tensor, num_attention_heads, attn_head_size): @classmethod def _merge_heads(cls, tensor, num_attention_heads, attn_head_size): - """ - Merges attn_head_size dim and num_attn_heads dim into hidden dim - """ + """Merges attn_head_size dim and num_attn_heads dim into hidden dim.""" # tensor [bs, num_attention_heads, seq_len, attn_head_size] tensor = tensor.permute(0, 2, 1, 3).contiguous() # -> [bs, seq_len, num_attention_heads, attn_head_size] @@ -280,5 +276,3 @@ def convert_kvcache_gpt_neox_heavy_recent(model, config): model._modules[name] = GPTNeoXAttention_Mask(config) return model - - diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py index 4ca213fa0a1..c3e3f48d4ed 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py @@ -19,7 +19,7 @@ import pdb import copy import math -import numpy as np +import numpy as np from dataclasses import dataclass from typing import Optional, Tuple, Union @@ -71,7 +71,7 @@ def local_heavy_hitter_mask(attn_weights, heavy_budget): class LlamaAttention_heavy_hitter(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" + """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__(self, config: LlamaConfig): super().__init__() @@ -148,7 +148,7 @@ def forward( heavy_budget = int(self.heavy_budget_ratio * attn_weights.shape[-1]) recent_budget = int(self.recent_budget_ratio * attn_weights.shape[-1]) - + # # Heavy Hitter Mask (Based on local statistics) # if heavy_budget > 0: # mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget) # Default: No padding applied to input @@ -165,10 +165,10 @@ def forward( # attn_weights[~mask_bottom] = torch.min(attention_mask) - + # Heavy Hitter Mask (Based on global statistics) tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(attn_weights.dtype) - tmp_sum = torch.sum(tmp_attn, dim=-2) + tmp_sum = torch.sum(tmp_attn, dim=-2) _, tmp_topk = tmp_sum.topk(k=heavy_budget, dim=-1) zeros = torch.zeros_like(tmp_sum, dtype=torch.bool) @@ -215,4 +215,3 @@ def convert_kvcache_llama_heavy_recent(model, config): model._modules[name] = LlamaAttention_heavy_hitter(config) return model - diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py index 23cd926a3eb..fce83989a4e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py @@ -1,8 +1,22 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import pdb import copy import math -import numpy as np +import numpy as np from dataclasses import dataclass from typing import Optional, Tuple, Union @@ -75,7 +89,7 @@ def sanity_check(mask): class OPTAttention_Mask(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" + """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__( self, @@ -270,4 +284,3 @@ def convert_kvcache_opt_heavy_recent(model, config): bias=config.enable_bias, ) return model - From c46ea7d046c5a42f9e8ee51d9cdb5a7db992dcca Mon Sep 17 00:00:00 2001 From: "biao.fang" Date: Tue, 23 Apr 2024 16:36:15 +0800 Subject: [PATCH 03/62] rebuild Signed-off-by: biao.fang --- .../h2o/run_lm_eval_harness.py | 31 +- .../__init__.py | 4 +- .../modeling/kv_cache_compression/h2o.py | 229 +++++ .../kv_cache_compression/models/__init__.py | 0 .../models/modeling_bloom.py | 219 +++++ .../models/modeling_gpt_neox.py} | 198 ++--- .../models/modeling_llama.py | 178 ++++ .../models/modeling_mistral.py | 163 ++++ .../models/modeling_opt.py} | 189 ++--- .../h2o_real_drop/modify_llama.py | 784 ------------------ .../h2o_sim_drop/__init__.py | 16 - .../h2o_sim_drop/modify_llama.py | 218 ----- 12 files changed, 938 insertions(+), 1291 deletions(-) rename intel_extension_for_transformers/transformers/modeling/{kv_cahe_compression/h2o_real_drop => kv_cache_compression}/__init__.py (90%) create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py rename intel_extension_for_transformers/transformers/modeling/{kv_cahe_compression/h2o_sim_drop/modify_gptneox.py => kv_cache_compression/models/modeling_gpt_neox.py} (56%) create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py rename intel_extension_for_transformers/transformers/modeling/{kv_cahe_compression/h2o_sim_drop/modify_opt.py => kv_cache_compression/models/modeling_opt.py} (55%) delete mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py delete mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py delete mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py b/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py index f2ae8fbfdf2..10741ae63a1 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py @@ -3,12 +3,15 @@ import torch import copy +import sys +sys.path.insert(0, '/root/hengguo/intel-extension-for-transformers') + from lm_eval import evaluator, tasks from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig -from intel_extension_for_transformers.transformers.modeling.kv_cahe_compression.h2o_sim_drop.modify_llama import convert_kvcache_llama_heavy_recent -from intel_extension_for_transformers.transformers.modeling.kv_cahe_compression.h2o_sim_drop.modify_opt import convert_kvcache_opt_heavy_recent -from intel_extension_for_transformers.transformers.modeling.kv_cahe_compression.h2o_sim_drop.modify_gptneox import convert_kvcache_gpt_neox_heavy_recent +from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o_sim_drop.modify_llama import convert_kvcache_llama_heavy_recent +from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o_sim_drop.modify_opt import convert_kvcache_opt_heavy_recent +from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o_sim_drop.modify_gptneox import convert_kvcache_gpt_neox_heavy_recent from tasks import EvalHarnessAdaptor @@ -24,12 +27,13 @@ prog = 'ProgramName', description = 'What the program does', epilog = 'Text at the bottom of help') - parser.add_argument('--task_name', type=str, default='hellaswag') + parser.add_argument("--tasks", nargs='+', default=["lambada_openai", + "hellaswag", "winogrande", "piqa", "wikitext"], + type=str, help="tasks list for accuracy validation") parser.add_argument('--num_fewshot', type=int, default=0) parser.add_argument('--enable_small_cache', action='store_true') parser.add_argument('--model_name', type=str, default='facebook/opt-350m') - parser.add_argument('--model_type', type=str, default='opt') parser.add_argument("--cache_dir", type=str, default=None) parser.add_argument("--heavy_ratio", type=float, default=0.1) @@ -70,7 +74,7 @@ def eval(self, batch): return out t = DryRunner() adaptor = EvalHarnessAdaptor(t, seq, batch_size, shrink=pe != "fixed") - result = evaluator.evaluate(adaptor, tasks.get_task_dict([args.task_name]), False, args.num_fewshot, None) + result = evaluator.evaluate(adaptor, tasks.get_task_dict(args.tasks), False, args.num_fewshot, None) model_name = args.model_name if 'cpu' in args.device: @@ -84,13 +88,14 @@ def eval(self, batch): if args.enable_small_cache: print('Enable Small Cache Size') - config.heavy_ratio = args.heavy_ratio - config.recent_ratio = args.recent_ratio - checkpoint = copy.deepcopy(model.state_dict()) - model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) - model.load_state_dict(checkpoint) + # checkpoint = copy.deepcopy(model.state_dict()) + # model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) + from intel_extension_for_transformers.transformers.modeling.kv_cache_compression import convert_model + model = convert_model(model, heavy_ratio=args.heavy_ratio, recent_ratio=args.recent_ratio, h2o_min_seqlen=0) + # model.load_state_dict(checkpoint) model = model.to(device) + print('using device: ', device) model.eval() # model.half().eval() @@ -205,7 +210,7 @@ def eval(self, batch): t = RealRunner(args) adaptor = EvalHarnessAdaptor(t, seq, batch_size, shrink=pe != "fixed") - results = evaluator.evaluate(adaptor, tasks.get_task_dict([args.task_name]), False, args.num_fewshot, None) + results = evaluator.evaluate(adaptor, tasks.get_task_dict(args.tasks), False, args.num_fewshot, None) dumped = json.dumps(results, indent=2) - print(dumped) \ No newline at end of file + print(dumped) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py similarity index 90% rename from intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py rename to intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py index db2d9048f2d..e779fd750e2 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/__init__.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py @@ -13,4 +13,6 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. + +from .h2o import convert_model \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py new file mode 100644 index 00000000000..577971e340d --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import OrderedDict +import importlib + +import torch +from torch import nn + +ATTENTION_MAPPING_NAMES = OrderedDict( + [ + ("opt", "OPTAttention") + ] +) + +def set_module(model, op_name, new_module): + """Set module with a given op name. + + Args: + model (object): the input model. + op_name (str): name of op. + new_module (object): the input model. + + Returns: + module (object). + """ + module = model + name_list = op_name.split(".") + for name in name_list[:-1]: + if hasattr(module, name): + module = getattr(module, name) + else: + module = module + setattr(module, name_list[-1], new_module) + +def get_module(model, op_name): + """Get module from model by key name. + + Args: + model (torch.nn.Module): original model + key (str): module name to be replaced + """ + module = model + name_list = op_name.split(".") + for name in name_list: + if hasattr(module, name): + module = getattr(module, name) + else: + module = module + return module + + +def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024): + device = model.device + model_type = model.config.model_type + model_name = model_type.replace("-", "_") + atten_cls = getattr(importlib.import_module(f".{model_name}.modeling_{model_name}", "transformers.models"), ATTENTION_MAPPING_NAMES[model_type]) + h2o_cls = getattr(importlib.import_module(f".models.modeling_{model_name}", "intel_extension_for_transformers.transformers.modeling.kv_cache_compression"), "H2O" + ATTENTION_MAPPING_NAMES[model_type]) + atten_layers = [] + for name, module in model.named_modules(): + if isinstance(module, atten_cls): + atten_layers.append(name) + + for layer_name in atten_layers: + module = get_module(model, layer_name) + module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen) + set_module(model, layer_name, module) + print(model) + print(f"heavy_ratio={heavy_ratio}, recent_ratio={recent_ratio}") + model = model.to(device) + return model + + +def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): + + # attn_weights (head, query, keys) + dtype_attn_weights = attn_weights.dtype + seq_length = attn_weights.shape[-1] + if no_padding_seq_length is None: + padding_length = 0 + else: + padding_length = seq_length - no_padding_seq_length + + offset = torch.finfo(attn_weights.dtype).min + tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) + + accumulated_attention_score = torch.sum(tmp_attn[:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) + accumulated_attention_score[:,heavy_budget+padding_length:] = 0 + + if padding_length > 0: + accumulated_attention_score[:,:padding_length] = 0 + + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + mask_bottom[:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True + + for token_index in range(heavy_budget+padding_length, seq_length): + + tmp_attn_index = nn.functional.softmax(attn_weights[:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) + zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) + mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) + mask_bottom_index[:, token_index] = True + + mask_bottom[:,token_index,:] = mask_bottom_index + accumulated_attention_score += tmp_attn_index + accumulated_attention_score = accumulated_attention_score * mask_bottom_index + + mask_bottom = torch.tril(mask_bottom, diagonal=0) + + return mask_bottom + + +def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): + heavy_budget = int(heavy_budget_ratio * attn_weights.shape[-1]) + recent_budget = int(recent_budget_ratio * attn_weights.shape[-1]) + if heavy_budget > 0: + mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) # Default: No padding applied to input + else: + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + + # Recent Mask + ones = torch.ones_like(attn_weights, dtype=torch.bool) + ones = torch.triu(ones, diagonal=-recent_budget) + mask_bottom = torch.logical_or(mask_bottom, ones) + + # Combine h2o+recent and apply casual mask + mask_bottom = torch.tril(mask_bottom, diagonal=0) + return mask_bottom + +class H2OKVCache: + def __init__( + self, + hh_size=4, + recent_size=512, + k_seq_dim=2, + v_seq_dim=2, + ): + self.hh_size = hh_size + self.recent_size = recent_size + self.cache_size = hh_size + recent_size + self.k_seq_dim = k_seq_dim + self.v_seq_dim = v_seq_dim + self.hh_score = None + + def __call__(self, past_key_values, attn_score_cache): + + self._update_hh_score(attn_score_cache) + + if past_key_values is None: + return None + seq_len = past_key_values[0].size(self.k_seq_dim) + if seq_len <= self.cache_size: + return past_key_values + + # hh-selection + bsz, num_heads, _, head_dim = past_key_values[0].shape + + select_hh_scores = self.hh_score[:, :seq_len - self.recent_size] + _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) + keep_topk = keep_topk.sort().values + + # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) + keep_recent = torch.arange(seq_len - self.recent_size, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) + keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) + + mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) + mask = mask.scatter(-1, keep_idx, 1) + + k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + + self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) + + return (k_hh_recent, v_hh_recent) + + def evict_for_space(self, past_key_values, num_coming): + if past_key_values is None: + return None + seq_len = past_key_values[0][0].size(self.k_seq_dim) + if seq_len + num_coming <= self.cache_size: + return past_key_values + + # hh-selection + bsz, num_heads, _, head_dim = past_key_values[0].shape + + select_hh_scores = self.hh_score[:, :seq_len - self.recent_size + num_coming] + _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) + keep_topk = keep_topk.sort().values + + # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) + keep_recent = torch.arange(seq_len - self.recent_size + num_coming, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) + keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) + + mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) + mask = mask.scatter(-1, keep_idx, 1) + + k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + + self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) + + return (k_hh_recent, v_hh_recent) + + def _update_hh_score(self, attn_score_cache): + + num_new_tokens = attn_score_cache.shape[2] + + if self.hh_score is None: + self.hh_score = attn_score_cache.sum(0).sum(1) + else: + attn_score_cache = attn_score_cache.sum(0).sum(1) + attn_score_cache[:, :-num_new_tokens] += self.hh_score + self.hh_score = attn_score_cache + + def _clean_scores(self): + self.hh_score = None \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py new file mode 100644 index 00000000000..d83222db94c --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from torch.nn import functional as F + +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ..h2o import get_hh_mask + +class BloomAttention(nn.Module): + def __init__( + self, + model, + config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + ): + super().__init__() + + self.pretraining_tp = config.pretraining_tp + self.slow_but_exact = config.slow_but_exact + + self.hidden_size = config.hidden_size + self.num_heads = config.n_head + self.head_dim = self.hidden_size // self.num_heads + self.split_size = self.hidden_size + self.hidden_dropout = config.hidden_dropout + + if self.head_dim * self.num_heads != self.hidden_size: + raise ValueError( + f"`hidden_size` must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:" + f" {self.num_heads})." + ) + + # Layer-wise attention scaling + self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim) + self.beta = 1.0 + + self.query_key_value = model.query_key_value + self.dense = model.dense + self.attention_dropout = model.attention_dropout + + # for h2o + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen + + def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory + storage as `fused_qkv` + + Args: + fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim] + + Returns: + query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim] + value: [batch_size, seq_length, num_heads, head_dim] + """ + batch_size, seq_length, three_times_hidden_size = fused_qkv.shape + fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads, 3, self.head_dim) + return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :] + + def _merge_heads(self, x: torch.Tensor) -> torch.Tensor: + """ + Merge heads together over the last dimension + + Args: + x (`torch.tensor`, *required*): [batch_size * num_heads, seq_length, head_dim] + + Returns: + torch.tensor: [batch_size, seq_length, num_heads * head_dim] + """ + # What we want to achieve is: + # batch_size * num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads * head_dim + batch_size_and_num_heads, seq_length, _ = x.shape + batch_size = batch_size_and_num_heads // self.num_heads + + # First view to decompose the batch size + # batch_size * num_heads, seq_length, head_dim -> batch_size, num_heads, seq_length, head_dim + x = x.view(batch_size, self.num_heads, seq_length, self.head_dim) + + # batch_size, num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads, head_dim + x = x.permute(0, 2, 1, 3) + + # batch_size, seq_length, num_heads, head_dim -> batch_size, seq_length, num_heads * head_dim + return x.reshape(batch_size, seq_length, self.num_heads * self.head_dim) + + def forward( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + alibi: torch.Tensor, + attention_mask: torch.Tensor, + layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + head_mask: Optional[torch.Tensor] = None, + use_cache: bool = False, + output_attentions: bool = False, + ): + fused_qkv = self.query_key_value(hidden_states) # [batch_size, seq_length, 3 x hidden_size] + + # 3 x [batch_size, seq_length, num_heads, head_dim] + (query_layer, key_layer, value_layer) = self._split_heads(fused_qkv) + + batch_size, q_length, _, _ = query_layer.shape + + query_layer = query_layer.transpose(1, 2).reshape(batch_size * self.num_heads, q_length, self.head_dim) + key_layer = key_layer.permute(0, 2, 3, 1).reshape(batch_size * self.num_heads, self.head_dim, q_length) + value_layer = value_layer.transpose(1, 2).reshape(batch_size * self.num_heads, q_length, self.head_dim) + if layer_past is not None: + past_key, past_value = layer_past + # concatenate along seq_length dimension: + # - key: [batch_size * self.num_heads, head_dim, kv_length] + # - value: [batch_size * self.num_heads, kv_length, head_dim] + key_layer = torch.cat((past_key, key_layer), dim=2) + value_layer = torch.cat((past_value, value_layer), dim=1) + + _, _, kv_length = key_layer.shape + + if use_cache is True: + present = (key_layer, value_layer) + else: + present = None + + # [batch_size * num_heads, q_length, kv_length] + # we use `torch.Tensor.baddbmm` instead of `torch.baddbmm` as the latter isn't supported by TorchScript v1.11 + matmul_result = alibi.baddbmm( + batch1=query_layer, + batch2=key_layer, + beta=self.beta, + alpha=self.inv_norm_factor, + ) + + # change view to [batch_size, num_heads, q_length, kv_length] + attention_scores = matmul_result.view(batch_size, self.num_heads, q_length, kv_length) + + # cast attention scores to fp32, compute scaled softmax and cast back to initial dtype - [batch_size, num_heads, q_length, kv_length] + input_dtype = attention_scores.dtype + # `float16` has a minimum value of -65504.0, whereas `bfloat16` and `float32` have a minimum value of `-3.4e+38` + if input_dtype == torch.float16: + attention_scores = attention_scores.to(torch.float) + attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.finfo(attention_scores.dtype).min) + # get hh mask + if q_length > self.h2o_min_seqlen: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.min(attention_mask) + + attention_probs = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(input_dtype) + + # [batch_size, num_heads, q_length, kv_length] + attention_probs = self.attention_dropout(attention_probs) + + if head_mask is not None: + attention_probs = attention_probs * head_mask + + # change view [batch_size x num_heads, q_length, kv_length] + attention_probs_reshaped = attention_probs.view(batch_size * self.num_heads, q_length, kv_length) + + # matmul: [batch_size * num_heads, q_length, head_dim] + context_layer = torch.bmm(attention_probs_reshaped, value_layer) + + # change view [batch_size, q_length, num_heads * head_dim] + context_layer = self._merge_heads(context_layer) + + # aggregate results across tp ranks. See here: https://github.com/pytorch/pytorch/issues/76232 + if self.pretraining_tp > 1 and self.slow_but_exact: + slices = self.hidden_size / self.pretraining_tp + output_tensor = torch.zeros_like(context_layer) + for i in range(self.pretraining_tp): + output_tensor = output_tensor + F.linear( + context_layer[:, :, int(i * slices) : int((i + 1) * slices)], + self.dense.weight[:, int(i * slices) : int((i + 1) * slices)], + ) + else: + output_tensor = self.dense(context_layer) + + output_tensor = dropout_add(output_tensor, residual, self.hidden_dropout, self.training) + + outputs = (output_tensor, present) + if output_attentions: + outputs += (attention_probs,) + + return outputs + +def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor: + """ + Dropout add function + + Args: + x (`torch.tensor`, *required*): + input tensor + residual (`torch.tensor`, *required*): + residual tensor + prob (`float`, *required*): + dropout probability + training (`bool`, *required*): + training mode + """ + out = F.dropout(x, p=prob, training=training) + out = residual + out + return out \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py similarity index 56% rename from intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py rename to intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py index 01bc7b63e87..d312b115382 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_gptneox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -14,106 +14,48 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import os -import pdb -import copy -import math -import numpy as np -from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import List, Optional, Tuple, Union import torch -from torch import nn -import torch.utils.checkpoint -import torch.nn.functional as F -from torch.cuda.amp import autocast -from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss - -from transformers.models.gpt_neox.modeling_gpt_neox import GPTNeoXRotaryEmbedding, GPTNeoXAttention, apply_rotary_pos_emb -# from transformers.models.gpt_neox.modeling_gpt_neox import GPTNeoXRotaryEmbedding - - -__all__ = ['convert_kvcache_gpt_neox_heavy_recent', 'GPTNeoXAttention_Mask'] - -def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): - - # attn_weights (BS, head, query, keys) - dtype_attn_weights = attn_weights.dtype - seq_length = attn_weights.shape[-1] - if no_padding_seq_length is None: - padding_length = 0 - else: - padding_length = seq_length - no_padding_seq_length - - offset = torch.finfo(attn_weights.dtype).min - tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) - - accumulated_attention_score = torch.sum(tmp_attn[:,:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) - accumulated_attention_score[:,:,heavy_budget+padding_length:] = 0 - if padding_length > 0: - accumulated_attention_score[:,:,:padding_length] = 0 +import torch.nn as nn - mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - mask_bottom[:,:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True +from ..h2o import get_hh_mask - for token_index in range(heavy_budget+padding_length, seq_length): - tmp_attn_index = nn.functional.softmax(attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) - _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) - zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) - mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) - mask_bottom_index[:,:, token_index] = True - - mask_bottom[:,:,token_index,:] = mask_bottom_index - accumulated_attention_score += tmp_attn_index - accumulated_attention_score = accumulated_attention_score * mask_bottom_index - - return mask_bottom - -def sanity_check(mask): - # mask (head, query, key) - ones = torch.ones_like(mask) - ones = torch.triu(ones, diagonal=0) - mask_bottom = torch.logical_or(mask, ones) - - error_cnt = 0 - for i in range(mask_bottom.shape[1]-1): - index = mask_bottom[:,i,:].eq(0).unsqueeze(1) - index[:,i:]=0 - error_cnt += (mask_bottom[:,i:,:] * index).sum().item() - print(error_cnt) - return error_cnt - -class GPTNeoXAttention_Mask(nn.Module): - def __init__(self, config): +class GPTNeoXAttention(nn.Module): + def __init__( + self, + model, + config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + ): super().__init__() + self.config = config self.num_attention_heads = config.num_attention_heads self.hidden_size = config.hidden_size + if self.hidden_size % self.num_attention_heads != 0: + raise ValueError( + "The hidden size is not divisble by the number of attention heads! Make sure to update them" + ) self.head_size = self.hidden_size // self.num_attention_heads self.rotary_ndims = int(self.head_size * config.rotary_pct) - max_positions = config.max_position_embeddings - self.register_buffer( - "bias", - torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view( - 1, 1, max_positions, max_positions - ), - ) - self.register_buffer("masked_bias", torch.tensor(-1e9)) - # self.rotary_emb = RotaryEmbedding( - self.rotary_emb = GPTNeoXRotaryEmbedding( - self.rotary_ndims, config.max_position_embeddings, base=config.rotary_emb_base - ) - self.register_buffer( - "norm_factor", - torch.sqrt(torch.tensor(self.head_size, dtype=torch.float32)).to(torch.get_default_dtype()), - persistent=False, - ) - self.query_key_value = nn.Linear(config.hidden_size, 3 * config.hidden_size) - self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.bias = model.bias - self.heavy_budget_ratio = config.heavy_ratio - self.recent_budget_ratio = config.recent_ratio + self.register_buffer("masked_bias", torch.tensor(-1e9), persistent=False) + self.rotary_emb = model.rotary_emb + + self.norm_factor = self.head_size**-0.5 + self.query_key_value = model.query_key_value + self.dense = model.dense + self.attention_dropout = model.attention_dropout + self.is_causal = True + + # for h2o + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen def forward( self, @@ -124,6 +66,7 @@ def forward( layer_past: Optional[Tuple[torch.Tensor]] = None, use_cache: Optional[bool] = False, output_attentions: Optional[bool] = False, + padding_mask: Optional[torch.Tensor] = None, ): has_layer_past = layer_past is not None @@ -209,6 +152,9 @@ def _attn(self, query, key, value, attention_mask=None, head_mask=None): batch_size, num_attention_heads, query_length, attn_head_size = query.size() key_length = key.size(-2) + # dynamically increase the causal mask with the key length, if needed. + if key_length > self.bias.shape[-1]: + self._init_bias(key_length, device=key.device) causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length] query = query.view(batch_size * num_attention_heads, query_length, attn_head_size) @@ -225,7 +171,7 @@ def _attn(self, query, key, value, attention_mask=None, head_mask=None): query, key.transpose(1, 2), beta=1.0, - alpha=(torch.tensor(1.0, dtype=self.norm_factor.dtype, device=self.norm_factor.device) / self.norm_factor), + alpha=self.norm_factor, ) attn_scores = attn_scores.view(batch_size, num_attention_heads, query_length, key_length) @@ -239,46 +185,52 @@ def _attn(self, query, key, value, attention_mask=None, head_mask=None): # Apply the attention mask attn_scores = attn_scores + attention_mask - #attn_scores (bs, head, token, token) - ### Heavy + Recent - heavy_budget = int(self.heavy_budget_ratio * attn_scores.shape[-1]) - recent_budget = int(self.recent_budget_ratio * attn_scores.shape[-1]) - - # Heavy Hitter Mask - if heavy_budget > 0: - mask_bottom = local_heavy_hitter_mask(attn_scores, heavy_budget, None) # Default: No padding applied to input - else: - mask_bottom = torch.zeros_like(attn_scores, dtype=torch.bool) - - ones = torch.ones_like(attn_scores, dtype=torch.bool) - ones = torch.triu(ones, diagonal=-recent_budget) - mask_bottom = torch.logical_or(mask_bottom, ones) - - mask_bottom = torch.tril(mask_bottom, diagonal=0) - - # mask_bottom = ones - attn_scores[~mask_bottom] = torch.finfo(attn_scores.dtype).min attn_weights = nn.functional.softmax(attn_scores, dim=-1) - attn_weights = attn_weights.to(value.dtype) + # get hh mask + if query_length > self.h2o_min_seqlen: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.min(attention_mask) + # Mask heads if we want to if head_mask is not None: attn_weights = attn_weights * head_mask + attn_weights = self.attention_dropout(attn_weights) + attn_output = torch.matmul(attn_weights, value) return attn_output, attn_weights -def convert_kvcache_gpt_neox_heavy_recent(model, config): - - for name, module in reversed(model._modules.items()): - - if len(list(module.children())) > 0: - model._modules[name] = convert_kvcache_gpt_neox_heavy_recent(module, config) - - if isinstance(module, GPTNeoXAttention): - model._modules[name] = GPTNeoXAttention_Mask(config) - - return model - - +def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`): + The position indices of the tokens corresponding to the query and key tensors. For example, this can be + used to pass offsetted position ids when working with a KV-cache. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos[position_ids].unsqueeze(unsqueeze_dim) + sin = sin[position_ids].unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py new file mode 100644 index 00000000000..ead4901687a --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch llama model.""" + +import math +from typing import List, Optional, Tuple, Union +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformers.cache_utils import Cache +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv + +from ..h2o import get_hh_mask, H2OKVCache + +logger = logging.get_logger(__name__) + +class H2OLlamaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + model, + config: LlamaConfig, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + real_drop=False + ): + super().__init__() + self.config = config + self.layer_idx = model.layer_idx + if self.layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + + self.q_proj = model.q_proj + self.k_proj = model.k_proj + self.v_proj = model.v_proj + self.o_proj = model.o_proj + self.rotary_emb = model.rotary_emb + + # for h2o + self.real_drop = real_drop + + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen + + self.h2o_kv_cache = H2OKVCache() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + if self.config.pretraining_tp > 1: + key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp + query_slices = self.q_proj.weight.split( + (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 + ) + key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) + value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) + + query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.config.pretraining_tp)] + query_states = torch.cat(query_states, dim=-1) + + key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.config.pretraining_tp)] + key_states = torch.cat(key_states, dim=-1) + + value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.config.pretraining_tp)] + value_states = torch.cat(value_states, dim=-1) + + else: + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + past_key_value = getattr(self, "past_key_value", past_key_value) + cos, sin = self.rotary_emb(value_states, position_ids) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # get hh mask + if q_len > self.h2o_min_seqlen: + if self.real_drop: + pass + else: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + if self.config.pretraining_tp > 1: + attn_output = attn_output.split(self.hidden_size // self.config.pretraining_tp, dim=2) + o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.config.pretraining_tp, dim=1) + attn_output = sum([F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.config.pretraining_tp)]) + else: + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py new file mode 100644 index 00000000000..967397c146e --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import logging +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from transformers.cache_utils import Cache +from transformers.models.mistral.modeling_mistral import apply_rotary_pos_emb, repeat_kv + +from ..h2o import get_hh_mask + +logger = logging.get_logger(__name__) + +class MistralAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__( + self, + model, + config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + ): + super().__init__() + self.config = config + self.layer_idx = model.layer_idx + if self.layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + self.attention_dropout = config.attention_dropout + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = model.q_proj + self.k_proj = model.k_proj + self.v_proj = model.v_proj + self.o_proj = model.o_proj + self.rotary_emb = model.rotary_emb + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if "padding_mask" in kwargs: + logger.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + attn_weights = attn_weights + attention_mask + + # get hh mask + if q_len > self.h2o_min_seqlen: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.min(attention_mask) + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py similarity index 55% rename from intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py rename to intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index 23cd926a3eb..018db0e8eca 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -1,113 +1,65 @@ -import os -import pdb -import copy -import math -import numpy as np -from dataclasses import dataclass -from typing import Optional, Tuple, Union +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch OPT model.""" +from typing import List, Optional, Tuple, Union import torch -from torch import nn -import torch.utils.checkpoint -import torch.nn.functional as F -from torch.cuda.amp import autocast -from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss +import torch.nn as nn -from transformers.models.opt.modeling_opt import OPTAttention +from ..h2o import get_hh_mask - -__all__ = ['convert_kvcache_opt_heavy_recent', 'OPTAttention_Mask'] - - - -def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): - - # attn_weights (head, query, keys) - dtype_attn_weights = attn_weights.dtype - seq_length = attn_weights.shape[-1] - if no_padding_seq_length is None: - padding_length = 0 - else: - padding_length = seq_length - no_padding_seq_length - - offset = torch.finfo(attn_weights.dtype).min - tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) - - accumulated_attention_score = torch.sum(tmp_attn[:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) - accumulated_attention_score[:,heavy_budget+padding_length:] = 0 - - if padding_length > 0: - accumulated_attention_score[:,:padding_length] = 0 - - mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - mask_bottom[:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True - - for token_index in range(heavy_budget+padding_length, seq_length): - - tmp_attn_index = nn.functional.softmax(attn_weights[:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) - _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) - zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) - mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) - mask_bottom_index[:, token_index] = True - - mask_bottom[:,token_index,:] = mask_bottom_index - accumulated_attention_score += tmp_attn_index - accumulated_attention_score = accumulated_attention_score * mask_bottom_index - - mask_bottom = torch.tril(mask_bottom, diagonal=0) - - return mask_bottom - - -def sanity_check(mask): - # mask (head, query, key) - ones = torch.ones_like(mask) - ones = torch.triu(ones, diagonal=0) - mask_bottom = torch.logical_or(mask, ones) - - error_cnt = 0 - for i in range(mask_bottom.shape[1]-1): - index = mask_bottom[:,i,:].eq(0).unsqueeze(1) - index[:,i:]=0 - error_cnt += (mask_bottom[:,i:,:] * index).sum().item() - print(error_cnt) - return error_cnt - - -class OPTAttention_Mask(nn.Module): +class H2OOPTAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" def __init__( self, - embed_dim: int, - num_heads: int, - heavy_ratio: float, - recent_ratio: float, - dropout: float = 0.0, - is_decoder: bool = False, - bias: bool = True, + model, + config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, ): super().__init__() - self.embed_dim = embed_dim - self.num_heads = num_heads - self.dropout = dropout - self.head_dim = embed_dim // num_heads + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.dropout = config.attention_dropout + self.enable_bias = config.enable_bias - if (self.head_dim * num_heads) != self.embed_dim: + self.head_dim = self.embed_dim // self.num_heads + self.is_causal = True + + if (self.head_dim * self.num_heads) != self.embed_dim: raise ValueError( f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" - f" and `num_heads`: {num_heads})." + f" and `num_heads`: {self.num_heads})." ) self.scaling = self.head_dim**-0.5 - self.is_decoder = is_decoder + self.is_decoder = model.is_decoder - self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.k_proj = model.k_proj + self.v_proj = model.v_proj + self.q_proj = model.q_proj + self.out_proj = model.out_proj - self.heavy_budget_ratio = heavy_ratio - self.recent_budget_ratio = recent_ratio + # for h2o + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -181,29 +133,15 @@ def forward( f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" ) attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask - attn_weights = torch.max(attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)) + attn_weights = torch.max( + attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min, device=attn_weights.device) + ) attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - ### Heavy + Recent - heavy_budget = int(self.heavy_budget_ratio * attn_weights.shape[-1]) - recent_budget = int(self.recent_budget_ratio * attn_weights.shape[-1]) - - # Heavy Hitter Mask - if heavy_budget > 0: - mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) # Default: No padding applied to input - else: - mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - - # Recent Mask - ones = torch.ones_like(attn_weights, dtype=torch.bool) - ones = torch.triu(ones, diagonal=-recent_budget) - mask_bottom = torch.logical_or(mask_bottom, ones) - - # Combine h2o+recent and apply casual mask - mask_bottom = torch.tril(mask_bottom, diagonal=0) - # mask_bottom = ones - attn_weights[~mask_bottom] = torch.min(attention_mask) - + # get hh mask + if tgt_len > self.h2o_min_seqlen: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.min(attention_mask) # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 if attn_weights.dtype == torch.float16: @@ -249,25 +187,4 @@ def forward( attn_output = self.out_proj(attn_output) - return attn_output, attn_weights_reshaped, past_key_value - - -def convert_kvcache_opt_heavy_recent(model, config): - - for name, module in reversed(model._modules.items()): - - if len(list(module.children())) > 0: - model._modules[name] = convert_kvcache_opt_heavy_recent(module, config) - - if isinstance(module, OPTAttention): - model._modules[name] = OPTAttention_Mask( - embed_dim=module.embed_dim, - num_heads=config.num_attention_heads, - heavy_ratio = config.heavy_ratio, - recent_ratio = config.recent_ratio, - dropout=config.attention_dropout, - is_decoder=True, - bias=config.enable_bias, - ) - return model - + return attn_output, attn_weights_reshaped, past_key_value \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py deleted file mode 100644 index 247cb4d25fd..00000000000 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_real_drop/modify_llama.py +++ /dev/null @@ -1,784 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import math -from typing import Optional, Tuple - -import pdb -import torch -from torch import nn -import torch.utils.checkpoint - -import torch.nn.functional as F - -# from transformers.models.llama.configuration_llama import LlamaConfig -from transformers.models.llama.modeling_llama import ( - LlamaAttention, - rotate_half, - apply_rotary_pos_emb, - LlamaRotaryEmbedding, - apply_rotary_pos_emb, - LlamaForCausalLM, - LlamaLinearScalingRotaryEmbedding, - LlamaDynamicNTKScalingRotaryEmbedding -) -import types - -__all__ = ["H2OLlamaForCausalLM", "H2OLlamaAttention", - 'H2OLlamaAttention_streaming', 'H2OLlamaForCausalLM_streaming'] - - -from transformers.configuration_utils import PretrainedConfig - -LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP = {} - -class LlamaConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA - model according to the specified arguments, defining the model architecture. Instantiating a configuration with the - defaults will yield a similar configuration to that of the LLaMA-7B. - - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. - - - Args: - vocab_size (`int`, *optional*, defaults to 32000): - Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the - `inputs_ids` passed when calling [`LlamaModel`] - hidden_size (`int`, *optional*, defaults to 4096): - Dimension of the hidden representations. - intermediate_size (`int`, *optional*, defaults to 11008): - Dimension of the MLP representations. - num_hidden_layers (`int`, *optional*, defaults to 32): - Number of hidden layers in the Transformer decoder. - num_attention_heads (`int`, *optional*, defaults to 32): - Number of attention heads for each attention layer in the Transformer decoder. - num_key_value_heads (`int`, *optional*): - This is the number of key_value heads that should be used to implement Grouped Query Attention. If - `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if - `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When - converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed - by meanpooling all the original heads within that group. For more details checkout [this - paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to - `num_attention_heads`. - hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): - The non-linear activation function (function or string) in the decoder. - max_position_embeddings (`int`, *optional*, defaults to 2048): - The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens, - Llama 2 up to 4096, CodeLlama up to 16384. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - rms_norm_eps (`float`, *optional*, defaults to 1e-06): - The epsilon used by the rms normalization layers. - use_cache (`bool`, *optional*, defaults to `True`): - Whether or not the model should return the last key/values attentions (not used by all models). Only - relevant if `config.is_decoder=True`. - pad_token_id (`int`, *optional*): - Padding token id. - bos_token_id (`int`, *optional*, defaults to 1): - Beginning of stream token id. - eos_token_id (`int`, *optional*, defaults to 2): - End of stream token id. - pretraining_tp (`int`, *optional*, defaults to 1): - Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this - document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is - necessary to ensure exact reproducibility of the pretraining results. Please refer to [this - issue](https://github.com/pytorch/pytorch/issues/76232). - tie_word_embeddings (`bool`, *optional*, defaults to `False`): - Whether to tie weight embeddings - rope_theta (`float`, *optional*, defaults to 10000.0): - The base period of the RoPE embeddings. - rope_scaling (`Dict`, *optional*): - Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling - strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is - `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update - `max_position_embeddings` to the expected new maximum. See the following thread for more information on how - these scaling strategies behave: - https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an - experimental feature, subject to breaking API changes in future versions. - attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`): - Whether to use a bias in the query, key, value and output projection layers during self-attention. - attention_dropout (`float`, *optional*, defaults to 0.0): - The dropout ratio for the attention probabilities. - - ```python - >>> from transformers import LlamaModel, LlamaConfig - - >>> # Initializing a LLaMA llama-7b style configuration - >>> configuration = LlamaConfig() - - >>> # Initializing a model from the llama-7b style configuration - >>> model = LlamaModel(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "llama" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - vocab_size=32000, - hidden_size=4096, - intermediate_size=11008, - num_hidden_layers=32, - num_attention_heads=32, - num_key_value_heads=None, - hidden_act="silu", - max_position_embeddings=2048, - initializer_range=0.02, - rms_norm_eps=1e-6, - use_cache=True, - pad_token_id=None, - bos_token_id=1, - eos_token_id=2, - pretraining_tp=1, - tie_word_embeddings=False, - rope_theta=10000.0, - rope_scaling=None, - attention_bias=False, - attention_dropout=0.0, - **kwargs, - ): - self.vocab_size = vocab_size - self.max_position_embeddings = max_position_embeddings - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.pretraining_tp = pretraining_tp - self.use_cache = use_cache - self.rope_theta = rope_theta - self.rope_scaling = rope_scaling - self._rope_scaling_validation() - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout - - super().__init__( - pad_token_id=pad_token_id, - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - def _rope_scaling_validation(self): - """ - Validate the `rope_scaling` configuration. - """ - if self.rope_scaling is None: - return - - if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2: - raise ValueError( - "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, " - f"got {self.rope_scaling}" - ) - rope_scaling_type = self.rope_scaling.get("type", None) - rope_scaling_factor = self.rope_scaling.get("factor", None) - if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]: - raise ValueError( - f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}" - ) - if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0: - raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}") - - -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, - num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) - """ - batch, num_key_value_heads, slen, head_dim = hidden_states.shape - if n_rep == 1: - return hidden_states - hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) - return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) - -def _make_causal_mask( - bsz: int, tgt_len: int, past_key_values_length: int, dtype: torch.dtype, device: torch.device): - """ - Make causal mask used for bi-directional self-attention. - """ - mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) - mask_cond = torch.arange(mask.size(-1), device=device) - mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) - mask = mask.to(dtype) - - if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) - return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) - - -def apply_rotary_pos_emb_single(x, cos, sin, position_ids): - # The first two dimensions of cos and sin are always 1, so we can `squeeze` them. - cos = cos.squeeze(1).squeeze(0) # [seq_len, dim] - sin = sin.squeeze(1).squeeze(0) # [seq_len, dim] - cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] - sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] - x_embed = (x * cos) + (rotate_half(x) * sin) - return x_embed - - -class H2OKVCache_LayerWise: - def __init__( - self, - hh_size=4, - recent_size=512, - k_seq_dim=2, - v_seq_dim=2, - ): - print(f"H2OKVCache-LayerWise: {hh_size}, {recent_size}") - self.hh_size = hh_size - self.recent_size = recent_size - self.cache_size = hh_size + recent_size - self.k_seq_dim = k_seq_dim - self.v_seq_dim = v_seq_dim - self.hh_score = None - - def __call__(self, past_key_values, attn_score_cache): - - self._update_hh_score(attn_score_cache) - - if past_key_values is None: - return None - seq_len = past_key_values[0].size(self.k_seq_dim) - if seq_len <= self.cache_size: - return past_key_values - - # hh-selection - bsz, num_heads, _, head_dim = past_key_values[0].shape - - select_hh_scores = self.hh_score[:, :seq_len - self.recent_size] - _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) - keep_topk = keep_topk.sort().values - - # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) - keep_recent = torch.arange(seq_len - self.recent_size, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) - keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) - - mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) - mask = mask.scatter(-1, keep_idx, 1) - - k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) - v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) - - self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) - - return (k_hh_recent, v_hh_recent) - - def evict_for_space(self, past_key_values, num_coming): - if past_key_values is None: - return None - seq_len = past_key_values[0][0].size(self.k_seq_dim) - if seq_len + num_coming <= self.cache_size: - return past_key_values - - # hh-selection - bsz, num_heads, _, head_dim = past_key_values[0].shape - - select_hh_scores = self.hh_score[:, :seq_len - self.recent_size + num_coming] - _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) - keep_topk = keep_topk.sort().values - - # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) - keep_recent = torch.arange(seq_len - self.recent_size + num_coming, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) - keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) - - mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) - mask = mask.scatter(-1, keep_idx, 1) - - k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) - v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) - - self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) - - return (k_hh_recent, v_hh_recent) - - def _update_hh_score(self, attn_score_cache): - - num_new_tokens = attn_score_cache.shape[2] - - if self.hh_score is None: - self.hh_score = attn_score_cache.sum(0).sum(1) - else: - attn_score_cache = attn_score_cache.sum(0).sum(1) - attn_score_cache[:, :-num_new_tokens] += self.hh_score - self.hh_score = attn_score_cache - - def _clean_scores(self): - self.hh_score = None - - -class H2OLlamaAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config: LlamaConfig): - super().__init__() - self.config = config - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.max_position_embeddings = config.max_position_embeddings - self.rope_theta = config.rope_theta - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError( - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads})." - ) - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) - self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) - self._init_rope() - - self.kv_cache = H2OKVCache_LayerWise( - hh_size=config.hh_size, - recent_size=config.recent_size, - k_seq_dim=2, - v_seq_dim=2, - ) - - def _init_rope(self): - if self.config.rope_scaling is None: - self.rotary_emb = LlamaRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - base=self.rope_theta, - ) - else: - scaling_type = self.config.rope_scaling["type"] - scaling_factor = self.config.rope_scaling["factor"] - if scaling_type == "linear": - self.rotary_emb = LlamaLinearScalingRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - scaling_factor=scaling_factor, - base=self.rope_theta, - ) - elif scaling_type == "dynamic": - self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - scaling_factor=scaling_factor, - base=self.rope_theta, - ) - else: - raise ValueError(f"Unknown RoPE scaling type {scaling_type}") - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def _clean_cache(self): - self.kv_cache._clean_scores() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - output_attentions: bool = False, - use_cache: bool = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - - bsz, q_len, _ = hidden_states.size() - - if self.config.pretraining_tp > 1: - key_value_slicing = ( - self.num_key_value_heads * self.head_dim - ) // self.config.pretraining_tp - query_slices = self.q_proj.weight.split( - (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 - ) - key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) - value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) - - query_states = [ - F.linear(hidden_states, query_slices[i]) - for i in range(self.config.pretraining_tp) - ] - query_states = torch.cat(query_states, dim=-1) - - key_states = [ - F.linear(hidden_states, key_slices[i]) - for i in range(self.config.pretraining_tp) - ] - key_states = torch.cat(key_states, dim=-1) - - value_states = [ - F.linear(hidden_states, value_slices[i]) - for i in range(self.config.pretraining_tp) - ] - value_states = torch.cat(value_states, dim=-1) - - else: - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view( - bsz, q_len, self.num_heads, self.head_dim - ).transpose(1, 2) - key_states = key_states.view( - bsz, q_len, self.num_key_value_heads, self.head_dim - ).transpose(1, 2) - value_states = value_states.view( - bsz, q_len, self.num_key_value_heads, self.head_dim - ).transpose(1, 2) - - # remake causal mask - attention_mask = _make_causal_mask( - bsz=bsz, - tgt_len=q_len, - past_key_values_length=past_key_value[0].shape[-2] if past_key_value is not None else 0, - dtype=query_states.dtype, - device=query_states.device, - ) - - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - kv_seq_len += past_key_value[0].shape[-2] - - position_length = kv_seq_len - if not position_ids.nelement() > 1: - if position_length < position_ids.item()+1: - position_length = position_ids.item()+1 - - cos, sin = self.rotary_emb(value_states, seq_len=position_length) - ### Shift Pos: query pos is min(cache_size, idx) - # query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - query_states = apply_rotary_pos_emb_single(query_states, cos, sin, position_ids) - key_states = apply_rotary_pos_emb_single(key_states, cos, sin, position_ids) - - if past_key_value is not None: - # reuse k, v, self_attention - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - - past_key_value = (key_states, value_states) if use_cache else None - - # repeat k/v heads if n_kv_heads < n_heads - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt( - self.head_dim - ) - - if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}" - ) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights + attention_mask - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to( - query_states.dtype - ) - - past_key_value = self.kv_cache(past_key_value, attn_weights.detach().clone()) - - attn_output = torch.matmul(attn_weights, value_states) - - if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) - - if self.config.pretraining_tp > 1: - attn_output = attn_output.split( - self.hidden_size // self.config.pretraining_tp, dim=2 - ) - o_proj_slices = self.o_proj.weight.split( - self.hidden_size // self.config.pretraining_tp, dim=1 - ) - attn_output = sum( - [ - F.linear(attn_output[i], o_proj_slices[i]) - for i in range(self.config.pretraining_tp) - ] - ) - else: - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - - -class H2OLlamaForCausalLM(LlamaForCausalLM): - def __init__(self, config): - super().__init__(config) - num_layers = len(self.model.layers) - for layer_idx in range(num_layers): - self.model.layers[layer_idx].self_attn = H2OLlamaAttention(config) - - - -## H2O KV Cache dropping with Position rolling -class H2OLlamaAttention_streaming(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config: LlamaConfig): - super().__init__() - self.config = config - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.max_position_embeddings = config.max_position_embeddings - self.rope_theta = config.rope_theta - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError( - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads})." - ) - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) - self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) - self._init_rope() - - self.kv_cache = H2OKVCache_LayerWise( - hh_size=config.hh_size, - recent_size=config.recent_size, - k_seq_dim=2, - v_seq_dim=2, - ) - - def _init_rope(self): - if self.config.rope_scaling is None: - self.rotary_emb = LlamaRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - base=self.rope_theta, - ) - else: - scaling_type = self.config.rope_scaling["type"] - scaling_factor = self.config.rope_scaling["factor"] - if scaling_type == "linear": - self.rotary_emb = LlamaLinearScalingRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - scaling_factor=scaling_factor, - base=self.rope_theta, - ) - elif scaling_type == "dynamic": - self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - scaling_factor=scaling_factor, - base=self.rope_theta, - ) - else: - raise ValueError(f"Unknown RoPE scaling type {scaling_type}") - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def _clean_cache(self): - self.kv_cache._clean_scores() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - output_attentions: bool = False, - use_cache: bool = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - - bsz, q_len, _ = hidden_states.size() - - if self.config.pretraining_tp > 1: - key_value_slicing = ( - self.num_key_value_heads * self.head_dim - ) // self.config.pretraining_tp - query_slices = self.q_proj.weight.split( - (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 - ) - key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) - value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) - - query_states = [ - F.linear(hidden_states, query_slices[i]) - for i in range(self.config.pretraining_tp) - ] - query_states = torch.cat(query_states, dim=-1) - - key_states = [ - F.linear(hidden_states, key_slices[i]) - for i in range(self.config.pretraining_tp) - ] - key_states = torch.cat(key_states, dim=-1) - - value_states = [ - F.linear(hidden_states, value_slices[i]) - for i in range(self.config.pretraining_tp) - ] - value_states = torch.cat(value_states, dim=-1) - - else: - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view( - bsz, q_len, self.num_heads, self.head_dim - ).transpose(1, 2) - key_states = key_states.view( - bsz, q_len, self.num_key_value_heads, self.head_dim - ).transpose(1, 2) - value_states = value_states.view( - bsz, q_len, self.num_key_value_heads, self.head_dim - ).transpose(1, 2) - - # remake causal mask - attention_mask = _make_causal_mask( - bsz=bsz, - tgt_len=q_len, - past_key_values_length=past_key_value[0].shape[-2] if past_key_value is not None else 0, - dtype=query_states.dtype, - device=query_states.device, - ) - - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - kv_seq_len += past_key_value[0].shape[-2] - - if not position_ids.nelement() > 1: - position_ids[0][0] = kv_seq_len - 1 - - cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) - ### Shift Pos: query pos is min(cache_size, idx) - # query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - query_states = apply_rotary_pos_emb_single(query_states, cos, sin, position_ids) - ### - - if past_key_value is not None: - # reuse k, v, self_attention - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - - past_key_value = (key_states, value_states) if use_cache else None - - ### Shift Pos: key pos is the pos in cache (Rolling KV Cache and using relative pos emb) - key_position_ids = torch.arange(kv_seq_len, device=position_ids.device).unsqueeze(0) - key_states = apply_rotary_pos_emb_single(key_states, cos, sin, key_position_ids) - ### - - # repeat k/v heads if n_kv_heads < n_heads - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt( - self.head_dim - ) - - if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}" - ) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights + attention_mask - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to( - query_states.dtype - ) - - past_key_value = self.kv_cache(past_key_value, attn_weights.detach().clone()) - - attn_output = torch.matmul(attn_weights, value_states) - - if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) - - if self.config.pretraining_tp > 1: - attn_output = attn_output.split( - self.hidden_size // self.config.pretraining_tp, dim=2 - ) - o_proj_slices = self.o_proj.weight.split( - self.hidden_size // self.config.pretraining_tp, dim=1 - ) - attn_output = sum( - [ - F.linear(attn_output[i], o_proj_slices[i]) - for i in range(self.config.pretraining_tp) - ] - ) - else: - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - - -class H2OLlamaForCausalLM_streaming(LlamaForCausalLM): - def __init__(self, config): - super().__init__(config) - num_layers = len(self.model.layers) - for layer_idx in range(num_layers): - self.model.layers[layer_idx].self_attn = H2OLlamaAttention_streaming(config) - - - - - diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py deleted file mode 100644 index db2d9048f2d..00000000000 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py deleted file mode 100644 index 4ca213fa0a1..00000000000 --- a/intel_extension_for_transformers/transformers/modeling/kv_cahe_compression/h2o_sim_drop/modify_llama.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pdb -import copy -import math -import numpy as np -from dataclasses import dataclass -from typing import Optional, Tuple, Union - -import torch -from torch import nn -import torch.utils.checkpoint -import torch.nn.functional as F -from torch.cuda.amp import autocast -from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss - - -from transformers.models.llama.configuration_llama import LlamaConfig -from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding, LlamaAttention, apply_rotary_pos_emb - - -__all__ = ['convert_kvcache_llama_heavy_recent', 'LlamaAttention_heavy_hitter'] - - -def local_heavy_hitter_mask(attn_weights, heavy_budget): - - # attn_weights (BS, head, query, keys) - dtype_attn_weights = attn_weights.dtype - seq_length = attn_weights.shape[-1] - padding_length = 0 - - offset = torch.finfo(attn_weights.dtype).min - tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) - - accumulated_attention_score = torch.sum(tmp_attn[:,:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) - accumulated_attention_score[:,:,heavy_budget+padding_length:] = 0 - accumulated_attention_score[:,:,:padding_length] = 0 - - mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - mask_bottom[:,:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True - - for token_index in range(heavy_budget+padding_length, seq_length): - - tmp_attn_index = nn.functional.softmax(attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) - _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) - zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) - mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) - mask_bottom_index[:,:, token_index] = True - - mask_bottom[:,:,token_index,:] = mask_bottom_index - accumulated_attention_score += tmp_attn_index - accumulated_attention_score = accumulated_attention_score * mask_bottom_index - - return mask_bottom - - -class LlamaAttention_heavy_hitter(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config: LlamaConfig): - super().__init__() - self.config = config - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.max_position_embeddings = config.max_position_embeddings - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError( - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads})." - ) - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) - self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) - self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) - self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) - self.rotary_emb = LlamaRotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings) - - self.heavy_budget_ratio = config.heavy_ratio - self.recent_budget_ratio = config.recent_ratio - - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - output_attentions: bool = False, - use_cache: bool = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - kv_seq_len += past_key_value[0].shape[-2] - cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - # [bsz, nh, t, hd] - - if past_key_value is not None: - # reuse k, v, self_attention - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - - past_key_value = (key_states, value_states) if use_cache else None - - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}" - ) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights + attention_mask - attn_weights = torch.max(attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)) - - ### Heavy + Recent - heavy_budget = int(self.heavy_budget_ratio * attn_weights.shape[-1]) - recent_budget = int(self.recent_budget_ratio * attn_weights.shape[-1]) - - - # # Heavy Hitter Mask (Based on local statistics) - # if heavy_budget > 0: - # mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget) # Default: No padding applied to input - # else: - # mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - - # ones = torch.ones_like(attn_weights, dtype=torch.bool) - # ones = torch.triu(ones, diagonal=-recent_budget) - # mask_bottom = torch.logical_or(mask_bottom, ones) - - # mask_bottom = torch.tril(mask_bottom, diagonal=0) - - # # mask_bottom = ones - # attn_weights[~mask_bottom] = torch.min(attention_mask) - - - - # Heavy Hitter Mask (Based on global statistics) - tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(attn_weights.dtype) - tmp_sum = torch.sum(tmp_attn, dim=-2) - _, tmp_topk = tmp_sum.topk(k=heavy_budget, dim=-1) - - zeros = torch.zeros_like(tmp_sum, dtype=torch.bool) - mask_bottom = zeros.scatter(-1, tmp_topk, True).unsqueeze(2) - mask_bottom = mask_bottom.expand(mask_bottom.shape[0], mask_bottom.shape[1], attn_weights.shape[-2], mask_bottom.shape[-1]) - - ones = torch.ones_like(attn_weights, dtype=torch.bool) - ones = torch.tril(ones, diagonal=recent_budget) - ones = torch.triu(ones, diagonal=-recent_budget) - mask_bottom = torch.logical_or(mask_bottom, ones) - # mask_bottom = ones - attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min - - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_output = torch.matmul(attn_weights, value_states) - - if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2) - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) - - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - - -def convert_kvcache_llama_heavy_recent(model, config): - - for name, module in reversed(model._modules.items()): - - if len(list(module.children())) > 0: - model._modules[name] = convert_kvcache_llama_heavy_recent(module, config) - - if isinstance(module, LlamaAttention): - model._modules[name] = LlamaAttention_heavy_hitter(config) - - return model - From 9d27733e1c1c32a8becdcdecc7cd5156fd32c872 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 08:39:01 +0000 Subject: [PATCH 04/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../modeling/kv_cache_compression/h2o.py | 10 +++++----- .../kv_cache_compression/models/__init__.py | 14 ++++++++++++++ .../kv_cache_compression/models/modeling_bloom.py | 11 ++++------- .../models/modeling_gpt_neox.py | 2 +- .../kv_cache_compression/models/modeling_llama.py | 8 ++++---- .../models/modeling_mistral.py | 11 ++++++----- .../kv_cache_compression/models/modeling_opt.py | 4 ++-- 7 files changed, 36 insertions(+), 24 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 577971e340d..84dfb529ac8 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -73,7 +73,7 @@ def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024): for name, module in model.named_modules(): if isinstance(module, atten_cls): atten_layers.append(name) - + for layer_name in atten_layers: module = get_module(model, layer_name) module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen) @@ -81,7 +81,7 @@ def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024): print(model) print(f"heavy_ratio={heavy_ratio}, recent_ratio={recent_ratio}") model = model.to(device) - return model + return model def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): @@ -129,8 +129,8 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): if heavy_budget > 0: mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) # Default: No padding applied to input else: - mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + # Recent Mask ones = torch.ones_like(attn_weights, dtype=torch.bool) ones = torch.triu(ones, diagonal=-recent_budget) @@ -226,4 +226,4 @@ def _update_hh_score(self, attn_score_cache): self.hh_score = attn_score_cache def _clean_scores(self): - self.hh_score = None \ No newline at end of file + self.hh_score = None diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py index e69de29bb2d..2045808acc2 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py index d83222db94c..0c725fc5b16 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -64,8 +64,7 @@ def __init__( self.h2o_min_seqlen = h2o_min_seqlen def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory + """Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory storage as `fused_qkv` Args: @@ -80,8 +79,7 @@ def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Ten return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :] def _merge_heads(self, x: torch.Tensor) -> torch.Tensor: - """ - Merge heads together over the last dimension + """Merge heads together over the last dimension. Args: x (`torch.tensor`, *required*): [batch_size * num_heads, seq_length, head_dim] @@ -201,8 +199,7 @@ def forward( return outputs def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor: - """ - Dropout add function + """Dropout add function. Args: x (`torch.tensor`, *required*): @@ -216,4 +213,4 @@ def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: """ out = F.dropout(x, p=prob, training=training) out = residual + out - return out \ No newline at end of file + return out diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py index 6478841f979..86b70cb1318 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -44,7 +44,7 @@ def __init__( self.hidden_size = config.hidden_size if self.hidden_size % self.num_attention_heads != 0: raise ValueError( - "The hidden size is not divisble by the number of attention heads! Make sure to update them" + "The hidden size is not divisible by the number of attention heads! Make sure to update them" ) self.head_size = self.hidden_size // self.num_attention_heads self.rotary_ndims = int(self.head_size * config.rotary_pct) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index ead4901687a..369a41c7c23 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" PyTorch llama model.""" +"""PyTorch llama model.""" import math from typing import List, Optional, Tuple, Union @@ -33,11 +33,11 @@ logger = logging.get_logger(__name__) class H2OLlamaAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" + """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__( self, - model, + model, config: LlamaConfig, heavy_ratio, recent_ratio, @@ -175,4 +175,4 @@ def forward( if not output_attentions: attn_weights = None - return attn_output, attn_weights, past_key_value \ No newline at end of file + return attn_output, attn_weights, past_key_value diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index 967397c146e..a7cb95207f4 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -29,14 +29,15 @@ logger = logging.get_logger(__name__) class MistralAttention(nn.Module): - """ - Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + """Multi-headed attention from 'Attention Is All You Need' paper. + + Modified to use sliding window attention: Longformer and "Generating Long Sequences with Sparse Transformers". """ def __init__( self, - model, + model, config, heavy_ratio, recent_ratio, @@ -135,7 +136,7 @@ def forward( ) attn_weights = attn_weights + attention_mask - + # get hh mask if q_len > self.h2o_min_seqlen: mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) @@ -160,4 +161,4 @@ def forward( if not output_attentions: attn_weights = None - return attn_output, attn_weights, past_key_value \ No newline at end of file + return attn_output, attn_weights, past_key_value diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index a75e883f710..077d8c5156f 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -10,7 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" PyTorch OPT model.""" +"""PyTorch OPT model.""" from typing import List, Optional, Tuple, Union import torch @@ -19,7 +19,7 @@ from ..h2o import get_hh_mask class H2OOPTAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" + """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__( self, From 444490de4b04f1d81e86ff668b065e8d74d1f9a6 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Thu, 25 Apr 2024 03:01:25 -0400 Subject: [PATCH 05/62] update Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/__init__.py | 18 ++ .../modeling/kv_cache_compression/h2o.py | 244 ++++++++++++++++++ .../kv_cache_compression/models/__init__.py | 0 .../models/modeling_bloom.py | 219 ++++++++++++++++ .../models/modeling_gpt_neox.py | 236 +++++++++++++++++ .../models/modeling_llama.py | 180 +++++++++++++ .../models/modeling_mistral.py | 168 ++++++++++++ .../models/modeling_opt.py | 190 ++++++++++++++ 8 files changed, 1255 insertions(+) create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py new file mode 100644 index 00000000000..e779fd750e2 --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .h2o import convert_model \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py new file mode 100644 index 00000000000..382ae301950 --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import OrderedDict +import importlib + +import torch +from torch import nn + +ATTENTION_MAPPING_NAMES = OrderedDict( + [ + ("opt", "OPTAttention"), + ("llama", "LlamaAttention"), + ("gpt_neox", "GPTNeoXAttention"), + ("mistral", "MistralAttention"), + ("bloom", "BloomAttention"), + ] +) + +def set_module(model, op_name, new_module): + """Set module with a given op name. + + Args: + model (object): the input model. + op_name (str): name of op. + new_module (object): the input model. + + Returns: + module (object). + """ + module = model + name_list = op_name.split(".") + for name in name_list[:-1]: + if hasattr(module, name): + module = getattr(module, name) + else: + module = module + setattr(module, name_list[-1], new_module) + +def get_module(model, op_name): + """Get module from model by key name. + + Args: + model (torch.nn.Module): original model + key (str): module name to be replaced + """ + module = model + name_list = op_name.split(".") + for name in name_list: + if hasattr(module, name): + module = getattr(module, name) + else: + module = module + return module + + +def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024): + device = model.device + model_type = model.config.model_type + model_name = model_type.replace("-", "_") + atten_cls = getattr(importlib.import_module(f".{model_name}.modeling_{model_name}", "transformers.models"), ATTENTION_MAPPING_NAMES[model_type]) + h2o_cls = getattr(importlib.import_module(f".models.modeling_{model_name}", "intel_extension_for_transformers.transformers.modeling.kv_cache_compression"), "H2O" + ATTENTION_MAPPING_NAMES[model_type]) + atten_layers = [] + for name, module in model.named_modules(): + if isinstance(module, atten_cls): + atten_layers.append(name) + + for layer_name in atten_layers: + module = get_module(model, layer_name) + module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen) + set_module(model, layer_name, module) + model = model.to(device) + return model + + +def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): + + # attn_weights (head, query, keys) or (BS, head, query, keys) + attn_shape_len = len(attn_weights.shape) + assert attn_shape_len in [3,4], "Wrong shape of attn_weights. Should be (head, query, keys) or (BS, head, query, keys)" + dtype_attn_weights = attn_weights.dtype + seq_length = attn_weights.shape[-1] + if no_padding_seq_length is None: + padding_length = 0 + else: + padding_length = seq_length - no_padding_seq_length + + offset = torch.finfo(attn_weights.dtype).min + tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) + + if len(attn_weights.shape) == 3: + accumulated_attention_score = torch.sum(tmp_attn[:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) + accumulated_attention_score[:,heavy_budget+padding_length:] = 0 + if padding_length > 0: + accumulated_attention_score[:,:padding_length] = 0 + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + mask_bottom[:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True + else: + accumulated_attention_score = torch.sum(tmp_attn[:,:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) + accumulated_attention_score[:,:,heavy_budget+padding_length:] = 0 + accumulated_attention_score[:,:,:padding_length] = 0 + + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + mask_bottom[:,:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True + for token_index in range(heavy_budget+padding_length, seq_length): + + if attn_shape_len == 3: + tmp_attn_index = nn.functional.softmax(attn_weights[:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + else: + tmp_attn_index = nn.functional.softmax(attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) + zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) + mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) + if attn_shape_len == 3: + mask_bottom_index[:, token_index] = True + mask_bottom[:,token_index,:] = mask_bottom_index + else: + mask_bottom_index[:,:, token_index] = True + mask_bottom[:,:,token_index,:] = mask_bottom_index + accumulated_attention_score += tmp_attn_index + accumulated_attention_score = accumulated_attention_score * mask_bottom_index + + mask_bottom = torch.tril(mask_bottom, diagonal=0) + + return mask_bottom + + +def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): + heavy_budget = int(heavy_budget_ratio * attn_weights.shape[-1]) + recent_budget = int(recent_budget_ratio * attn_weights.shape[-1]) + if heavy_budget > 0: + mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) # Default: No padding applied to input + else: + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + + # Recent Mask + ones = torch.ones_like(attn_weights, dtype=torch.bool) + ones = torch.triu(ones, diagonal=-recent_budget) + mask_bottom = torch.logical_or(mask_bottom, ones) + + # Combine h2o+recent and apply casual mask + mask_bottom = torch.tril(mask_bottom, diagonal=0) + return mask_bottom + +class H2OKVCache: + def __init__( + self, + hh_size=4, + recent_size=512, + k_seq_dim=2, + v_seq_dim=2, + ): + self.hh_size = hh_size + self.recent_size = recent_size + self.cache_size = hh_size + recent_size + self.k_seq_dim = k_seq_dim + self.v_seq_dim = v_seq_dim + self.hh_score = None + + def __call__(self, past_key_values, attn_score_cache): + + self._update_hh_score(attn_score_cache) + + if past_key_values is None: + return None + seq_len = past_key_values[0].size(self.k_seq_dim) + if seq_len <= self.cache_size: + return past_key_values + + # hh-selection + bsz, num_heads, _, head_dim = past_key_values[0].shape + + select_hh_scores = self.hh_score[:, :seq_len - self.recent_size] + _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) + keep_topk = keep_topk.sort().values + + # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) + keep_recent = torch.arange(seq_len - self.recent_size, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) + keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) + + mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) + mask = mask.scatter(-1, keep_idx, 1) + + k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + + self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) + + return (k_hh_recent, v_hh_recent) + + def evict_for_space(self, past_key_values, num_coming): + if past_key_values is None: + return None + seq_len = past_key_values[0][0].size(self.k_seq_dim) + if seq_len + num_coming <= self.cache_size: + return past_key_values + + # hh-selection + bsz, num_heads, _, head_dim = past_key_values[0].shape + + select_hh_scores = self.hh_score[:, :seq_len - self.recent_size + num_coming] + _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) + keep_topk = keep_topk.sort().values + + # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) + keep_recent = torch.arange(seq_len - self.recent_size + num_coming, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) + keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) + + mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) + mask = mask.scatter(-1, keep_idx, 1) + + k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + + self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) + + return (k_hh_recent, v_hh_recent) + + def _update_hh_score(self, attn_score_cache): + + num_new_tokens = attn_score_cache.shape[2] + + if self.hh_score is None: + self.hh_score = attn_score_cache.sum(0).sum(1) + else: + attn_score_cache = attn_score_cache.sum(0).sum(1) + attn_score_cache[:, :-num_new_tokens] += self.hh_score + self.hh_score = attn_score_cache + + def _clean_scores(self): + self.hh_score = None \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py new file mode 100644 index 00000000000..3d5cb92c875 --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from torch.nn import functional as F + +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ..h2o import get_hh_mask + +class H2OBloomAttention(nn.Module): + def __init__( + self, + model, + config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + ): + super().__init__() + + self.pretraining_tp = config.pretraining_tp + self.slow_but_exact = config.slow_but_exact + + self.hidden_size = config.hidden_size + self.num_heads = config.n_head + self.head_dim = self.hidden_size // self.num_heads + self.split_size = self.hidden_size + self.hidden_dropout = config.hidden_dropout + + if self.head_dim * self.num_heads != self.hidden_size: + raise ValueError( + f"`hidden_size` must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:" + f" {self.num_heads})." + ) + + # Layer-wise attention scaling + self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim) + self.beta = 1.0 + + self.query_key_value = model.query_key_value + self.dense = model.dense + self.attention_dropout = model.attention_dropout + + # for h2o + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen + + def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory + storage as `fused_qkv` + + Args: + fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim] + + Returns: + query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim] + value: [batch_size, seq_length, num_heads, head_dim] + """ + batch_size, seq_length, three_times_hidden_size = fused_qkv.shape + fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads, 3, self.head_dim) + return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :] + + def _merge_heads(self, x: torch.Tensor) -> torch.Tensor: + """ + Merge heads together over the last dimension + + Args: + x (`torch.tensor`, *required*): [batch_size * num_heads, seq_length, head_dim] + + Returns: + torch.tensor: [batch_size, seq_length, num_heads * head_dim] + """ + # What we want to achieve is: + # batch_size * num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads * head_dim + batch_size_and_num_heads, seq_length, _ = x.shape + batch_size = batch_size_and_num_heads // self.num_heads + + # First view to decompose the batch size + # batch_size * num_heads, seq_length, head_dim -> batch_size, num_heads, seq_length, head_dim + x = x.view(batch_size, self.num_heads, seq_length, self.head_dim) + + # batch_size, num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads, head_dim + x = x.permute(0, 2, 1, 3) + + # batch_size, seq_length, num_heads, head_dim -> batch_size, seq_length, num_heads * head_dim + return x.reshape(batch_size, seq_length, self.num_heads * self.head_dim) + + def forward( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + alibi: torch.Tensor, + attention_mask: torch.Tensor, + layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + head_mask: Optional[torch.Tensor] = None, + use_cache: bool = False, + output_attentions: bool = False, + ): + fused_qkv = self.query_key_value(hidden_states) # [batch_size, seq_length, 3 x hidden_size] + + # 3 x [batch_size, seq_length, num_heads, head_dim] + (query_layer, key_layer, value_layer) = self._split_heads(fused_qkv) + + batch_size, q_length, _, _ = query_layer.shape + + query_layer = query_layer.transpose(1, 2).reshape(batch_size * self.num_heads, q_length, self.head_dim) + key_layer = key_layer.permute(0, 2, 3, 1).reshape(batch_size * self.num_heads, self.head_dim, q_length) + value_layer = value_layer.transpose(1, 2).reshape(batch_size * self.num_heads, q_length, self.head_dim) + if layer_past is not None: + past_key, past_value = layer_past + # concatenate along seq_length dimension: + # - key: [batch_size * self.num_heads, head_dim, kv_length] + # - value: [batch_size * self.num_heads, kv_length, head_dim] + key_layer = torch.cat((past_key, key_layer), dim=2) + value_layer = torch.cat((past_value, value_layer), dim=1) + + _, _, kv_length = key_layer.shape + + if use_cache is True: + present = (key_layer, value_layer) + else: + present = None + + # [batch_size * num_heads, q_length, kv_length] + # we use `torch.Tensor.baddbmm` instead of `torch.baddbmm` as the latter isn't supported by TorchScript v1.11 + matmul_result = alibi.baddbmm( + batch1=query_layer, + batch2=key_layer, + beta=self.beta, + alpha=self.inv_norm_factor, + ) + + # change view to [batch_size, num_heads, q_length, kv_length] + attention_scores = matmul_result.view(batch_size, self.num_heads, q_length, kv_length) + + # cast attention scores to fp32, compute scaled softmax and cast back to initial dtype - [batch_size, num_heads, q_length, kv_length] + input_dtype = attention_scores.dtype + # `float16` has a minimum value of -65504.0, whereas `bfloat16` and `float32` have a minimum value of `-3.4e+38` + if input_dtype == torch.float16: + attention_scores = attention_scores.to(torch.float) + attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.finfo(attention_scores.dtype).min) + # get hh mask + if q_length > self.h2o_min_seqlen: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.min(attention_mask) + + attention_probs = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(input_dtype) + + # [batch_size, num_heads, q_length, kv_length] + attention_probs = self.attention_dropout(attention_probs) + + if head_mask is not None: + attention_probs = attention_probs * head_mask + + # change view [batch_size x num_heads, q_length, kv_length] + attention_probs_reshaped = attention_probs.view(batch_size * self.num_heads, q_length, kv_length) + + # matmul: [batch_size * num_heads, q_length, head_dim] + context_layer = torch.bmm(attention_probs_reshaped, value_layer) + + # change view [batch_size, q_length, num_heads * head_dim] + context_layer = self._merge_heads(context_layer) + + # aggregate results across tp ranks. See here: https://github.com/pytorch/pytorch/issues/76232 + if self.pretraining_tp > 1 and self.slow_but_exact: + slices = self.hidden_size / self.pretraining_tp + output_tensor = torch.zeros_like(context_layer) + for i in range(self.pretraining_tp): + output_tensor = output_tensor + F.linear( + context_layer[:, :, int(i * slices) : int((i + 1) * slices)], + self.dense.weight[:, int(i * slices) : int((i + 1) * slices)], + ) + else: + output_tensor = self.dense(context_layer) + + output_tensor = dropout_add(output_tensor, residual, self.hidden_dropout, self.training) + + outputs = (output_tensor, present) + if output_attentions: + outputs += (attention_probs,) + + return outputs + +def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor: + """ + Dropout add function + + Args: + x (`torch.tensor`, *required*): + input tensor + residual (`torch.tensor`, *required*): + residual tensor + prob (`float`, *required*): + dropout probability + training (`bool`, *required*): + training mode + """ + out = F.dropout(x, p=prob, training=training) + out = residual + out + return out \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py new file mode 100644 index 00000000000..d312b115382 --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ..h2o import get_hh_mask + + +class GPTNeoXAttention(nn.Module): + def __init__( + self, + model, + config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + ): + super().__init__() + self.config = config + self.num_attention_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + if self.hidden_size % self.num_attention_heads != 0: + raise ValueError( + "The hidden size is not divisble by the number of attention heads! Make sure to update them" + ) + self.head_size = self.hidden_size // self.num_attention_heads + self.rotary_ndims = int(self.head_size * config.rotary_pct) + self.bias = model.bias + + self.register_buffer("masked_bias", torch.tensor(-1e9), persistent=False) + self.rotary_emb = model.rotary_emb + + self.norm_factor = self.head_size**-0.5 + self.query_key_value = model.query_key_value + self.dense = model.dense + self.attention_dropout = model.attention_dropout + self.is_causal = True + + # for h2o + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen + + def forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: torch.FloatTensor, + position_ids: torch.LongTensor, + head_mask: Optional[torch.FloatTensor] = None, + layer_past: Optional[Tuple[torch.Tensor]] = None, + use_cache: Optional[bool] = False, + output_attentions: Optional[bool] = False, + padding_mask: Optional[torch.Tensor] = None, + ): + has_layer_past = layer_past is not None + + # Compute QKV + # Attention heads [batch, seq_len, hidden_size] + # --> [batch, seq_len, (np * 3 * head_size)] + qkv = self.query_key_value(hidden_states) + + # [batch, seq_len, (num_heads * 3 * head_size)] + # --> [batch, seq_len, num_heads, 3 * head_size] + new_qkv_shape = qkv.size()[:-1] + (self.num_attention_heads, 3 * self.head_size) + qkv = qkv.view(*new_qkv_shape) + + # [batch, seq_len, num_attention_heads, 3 * head_size] --> 3 [batch, num_attention_heads, seq_len, head_size] + query = qkv[..., : self.head_size].permute(0, 2, 1, 3) + key = qkv[..., self.head_size : 2 * self.head_size].permute(0, 2, 1, 3) + value = qkv[..., 2 * self.head_size :].permute(0, 2, 1, 3) + + # Compute rotary embeddings on rotary_ndims + query_rot = query[..., : self.rotary_ndims] + query_pass = query[..., self.rotary_ndims :] + key_rot = key[..., : self.rotary_ndims] + key_pass = key[..., self.rotary_ndims :] + + # Compute token offset for rotary embeddings (when decoding) + seq_len = key.shape[-2] + if has_layer_past: + seq_len += layer_past[0].shape[-2] + cos, sin = self.rotary_emb(value, seq_len=seq_len) + query, key = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids) + query = torch.cat((query, query_pass), dim=-1) + key = torch.cat((key, key_pass), dim=-1) + + # Cache QKV values + if has_layer_past: + past_key = layer_past[0] + past_value = layer_past[1] + key = torch.cat((past_key, key), dim=-2) + value = torch.cat((past_value, value), dim=-2) + present = (key, value) if use_cache else None + + # Compute attention + attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask) + + # Reshape outputs + attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_size) + attn_output = self.dense(attn_output) + + outputs = (attn_output, present) + if output_attentions: + outputs += (attn_weights,) + + return outputs + + @classmethod + def _split_heads(cls, tensor, num_attention_heads, attn_head_size): + """ + Splits hidden dim into attn_head_size and num_attention_heads + """ + # tensor: [bs, seq_len, hidden_size] + new_shape = tensor.size()[:-1] + (num_attention_heads, attn_head_size) + # -> [bs, seq_len, num_attention_heads, attn_head_size] + tensor = tensor.view(new_shape) + # -> [bs, num_attention_heads, seq_len, attn_head_size] + tensor = tensor.permute(0, 2, 1, 3) + return tensor + + @classmethod + def _merge_heads(cls, tensor, num_attention_heads, attn_head_size): + """ + Merges attn_head_size dim and num_attn_heads dim into hidden dim + """ + # tensor [bs, num_attention_heads, seq_len, attn_head_size] + tensor = tensor.permute(0, 2, 1, 3).contiguous() + # -> [bs, seq_len, num_attention_heads, attn_head_size] + tensor = tensor.view(tensor.size(0), tensor.size(1), num_attention_heads * attn_head_size) + # -> [bs, seq_len, hidden_size] + return tensor + + def _attn(self, query, key, value, attention_mask=None, head_mask=None): + # q, k, v: [bs, num_attention_heads, seq_len, attn_head_size] + # compute causal mask from causal mask buffer + batch_size, num_attention_heads, query_length, attn_head_size = query.size() + key_length = key.size(-2) + + # dynamically increase the causal mask with the key length, if needed. + if key_length > self.bias.shape[-1]: + self._init_bias(key_length, device=key.device) + causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length] + + query = query.view(batch_size * num_attention_heads, query_length, attn_head_size) + key = key.view(batch_size * num_attention_heads, key_length, attn_head_size) + attn_scores = torch.zeros( + batch_size * num_attention_heads, + query_length, + key_length, + dtype=query.dtype, + device=key.device, + ) + attn_scores = torch.baddbmm( + attn_scores, + query, + key.transpose(1, 2), + beta=1.0, + alpha=self.norm_factor, + ) + attn_scores = attn_scores.view(batch_size, num_attention_heads, query_length, key_length) + + mask_value = torch.finfo(attn_scores.dtype).min + # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`. + # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device` + mask_value = torch.tensor(mask_value, dtype=attn_scores.dtype).to(attn_scores.device) + attn_scores = torch.where(causal_mask, attn_scores, mask_value) + + if attention_mask is not None: + # Apply the attention mask + attn_scores = attn_scores + attention_mask + + attn_weights = nn.functional.softmax(attn_scores, dim=-1) + attn_weights = attn_weights.to(value.dtype) + + # get hh mask + if query_length > self.h2o_min_seqlen: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.min(attention_mask) + + # Mask heads if we want to + if head_mask is not None: + attn_weights = attn_weights * head_mask + + attn_weights = self.attention_dropout(attn_weights) + + attn_output = torch.matmul(attn_weights, value) + return attn_output, attn_weights + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`): + The position indices of the tokens corresponding to the query and key tensors. For example, this can be + used to pass offsetted position ids when working with a KV-cache. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos[position_ids].unsqueeze(unsqueeze_dim) + sin = sin[position_ids].unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py new file mode 100644 index 00000000000..e550e56f929 --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch llama model.""" + +import math +from typing import List, Optional, Tuple, Union +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformers.cache_utils import Cache +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv + +from ..h2o import get_hh_mask, H2OKVCache + +logger = logging.getLogger(__name__) + +class H2OLlamaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + model, + config: LlamaConfig, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + real_drop=False + ): + super().__init__() + self.config = config + self.layer_idx = model.layer_idx + if self.layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + + self.q_proj = model.q_proj + self.k_proj = model.k_proj + self.v_proj = model.v_proj + self.o_proj = model.o_proj + self.rotary_emb = model.rotary_emb + + # for h2o + self.real_drop = real_drop + + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen + + self.h2o_kv_cache = H2OKVCache() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + if self.config.pretraining_tp > 1: + key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp + query_slices = self.q_proj.weight.split( + (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 + ) + key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) + value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) + + query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.config.pretraining_tp)] + query_states = torch.cat(query_states, dim=-1) + + key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.config.pretraining_tp)] + key_states = torch.cat(key_states, dim=-1) + + value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.config.pretraining_tp)] + value_states = torch.cat(value_states, dim=-1) + + else: + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + past_key_value = getattr(self, "past_key_value", past_key_value) + cos, sin = self.rotary_emb(value_states, position_ids) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask + if cache_position is not None: + causal_mask = attention_mask[:, :, cache_position, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # get hh mask + if q_len > self.h2o_min_seqlen: + if self.real_drop: + pass + else: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + if self.config.pretraining_tp > 1: + attn_output = attn_output.split(self.hidden_size // self.config.pretraining_tp, dim=2) + o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.config.pretraining_tp, dim=1) + attn_output = sum([F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.config.pretraining_tp)]) + else: + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py new file mode 100644 index 00000000000..b94b05e0927 --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import logging +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from transformers.cache_utils import Cache +from transformers.models.mistral.modeling_mistral import apply_rotary_pos_emb, repeat_kv + +from ..h2o import get_hh_mask + +logger = logging.getLogger(__name__) + +class H2OMistralAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__( + self, + model, + config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + ): + super().__init__() + self.config = config + self.layer_idx = model.layer_idx + if self.layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + self.attention_dropout = config.attention_dropout + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = model.q_proj + self.k_proj = model.k_proj + self.v_proj = model.v_proj + self.o_proj = model.o_proj + self.rotary_emb = model.rotary_emb + + # for h2o + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if "padding_mask" in kwargs: + logger.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + attn_weights = attn_weights + attention_mask + + # get hh mask + if q_len > self.h2o_min_seqlen: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.min(attention_mask) + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py new file mode 100644 index 00000000000..018db0e8eca --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch OPT model.""" +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ..h2o import get_hh_mask + +class H2OOPTAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + model, + config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + ): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.dropout = config.attention_dropout + self.enable_bias = config.enable_bias + + self.head_dim = self.embed_dim // self.num_heads + self.is_causal = True + + if (self.head_dim * self.num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {self.num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.is_decoder = model.is_decoder + + self.k_proj = model.k_proj + self.v_proj = model.v_proj + self.q_proj = model.q_proj + self.out_proj = model.out_proj + + # for h2o + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = torch.max( + attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min, device=attn_weights.device) + ) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # get hh mask + if tgt_len > self.h2o_min_seqlen: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.min(attention_mask) + + # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 + if attn_weights.dtype == torch.float16: + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(torch.float16) + else: + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if layer_head_mask is not None: + if layer_head_mask.size() != (self.num_heads,): + raise ValueError( + f"Head mask for a single layer should be of size {(self.num_heads,)}, but is" + f" {layer_head_mask.size()}" + ) + attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if output_attentions: + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to be reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + + # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # partitioned aross GPUs when using tensor-parallelism. + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped, past_key_value \ No newline at end of file From 43090892ce9d0c66a9b77836fd041f0a0fa2f00d Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Thu, 25 Apr 2024 03:01:49 -0400 Subject: [PATCH 06/62] update Signed-off-by: n1ck-guo --- .../h2o/run_lm_eval_harness.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py b/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py index f2ae8fbfdf2..5988a090818 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py @@ -2,13 +2,19 @@ import json, tqdm import torch import copy +import os +os.environ["TOKENIZERS_PARALLELISM"] = "false" + +import sys +sys.path.insert(0, '/home/hengguo/code/intel-extension-for-transformers') from lm_eval import evaluator, tasks from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig -from intel_extension_for_transformers.transformers.modeling.kv_cahe_compression.h2o_sim_drop.modify_llama import convert_kvcache_llama_heavy_recent -from intel_extension_for_transformers.transformers.modeling.kv_cahe_compression.h2o_sim_drop.modify_opt import convert_kvcache_opt_heavy_recent -from intel_extension_for_transformers.transformers.modeling.kv_cahe_compression.h2o_sim_drop.modify_gptneox import convert_kvcache_gpt_neox_heavy_recent +from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o_sim_drop.modify_llama import convert_kvcache_llama_heavy_recent +from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o_sim_drop.modify_opt import convert_kvcache_opt_heavy_recent +from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o_sim_drop.modify_gptneox import convert_kvcache_gpt_neox_heavy_recent + from tasks import EvalHarnessAdaptor @@ -24,12 +30,13 @@ prog = 'ProgramName', description = 'What the program does', epilog = 'Text at the bottom of help') - parser.add_argument('--task_name', type=str, default='hellaswag') + parser.add_argument("--tasks", nargs='+', default=["lambada_openai", + "hellaswag", "winogrande", "piqa", "wikitext"], + type=str, help="tasks list for accuracy validation") parser.add_argument('--num_fewshot', type=int, default=0) parser.add_argument('--enable_small_cache', action='store_true') parser.add_argument('--model_name', type=str, default='facebook/opt-350m') - parser.add_argument('--model_type', type=str, default='opt') parser.add_argument("--cache_dir", type=str, default=None) parser.add_argument("--heavy_ratio", type=float, default=0.1) @@ -70,7 +77,7 @@ def eval(self, batch): return out t = DryRunner() adaptor = EvalHarnessAdaptor(t, seq, batch_size, shrink=pe != "fixed") - result = evaluator.evaluate(adaptor, tasks.get_task_dict([args.task_name]), False, args.num_fewshot, None) + result = evaluator.evaluate(adaptor, tasks.get_task_dict(args.tasks), False, args.num_fewshot, None) model_name = args.model_name if 'cpu' in args.device: @@ -84,13 +91,14 @@ def eval(self, batch): if args.enable_small_cache: print('Enable Small Cache Size') - config.heavy_ratio = args.heavy_ratio - config.recent_ratio = args.recent_ratio - checkpoint = copy.deepcopy(model.state_dict()) - model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) - model.load_state_dict(checkpoint) + # checkpoint = copy.deepcopy(model.state_dict()) + # model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) + from intel_extension_for_transformers.transformers.modeling.kv_cache_compression import convert_model + model = convert_model(model, heavy_ratio=args.heavy_ratio, recent_ratio=args.recent_ratio, h2o_min_seqlen=0) + # model.load_state_dict(checkpoint) model = model.to(device) + print('using device: ', device) model.eval() # model.half().eval() @@ -205,7 +213,7 @@ def eval(self, batch): t = RealRunner(args) adaptor = EvalHarnessAdaptor(t, seq, batch_size, shrink=pe != "fixed") - results = evaluator.evaluate(adaptor, tasks.get_task_dict([args.task_name]), False, args.num_fewshot, None) + results = evaluator.evaluate(adaptor, tasks.get_task_dict(args.tasks), False, args.num_fewshot, None) dumped = json.dumps(results, indent=2) - print(dumped) \ No newline at end of file + print(dumped) From 1b83e52413541addb638038f6e41c68163247b81 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 07:11:44 +0000 Subject: [PATCH 07/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../transformers/modeling/kv_cache_compression/h2o.py | 8 ++++---- .../modeling/kv_cache_compression/models/__init__.py | 1 - .../kv_cache_compression/models/modeling_bloom.py | 9 +++------ .../kv_cache_compression/models/modeling_gpt_neox.py | 10 +++------- .../kv_cache_compression/models/modeling_llama.py | 6 +++--- .../kv_cache_compression/models/modeling_mistral.py | 9 +++++---- .../kv_cache_compression/models/modeling_opt.py | 4 ++-- 7 files changed, 20 insertions(+), 27 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index fc9c53c3fbc..589abb8642c 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -77,13 +77,13 @@ def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024): for name, module in model.named_modules(): if isinstance(module, atten_cls): atten_layers.append(name) - + for layer_name in atten_layers: module = get_module(model, layer_name) module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen) set_module(model, layer_name, module) model = model.to(device) - return model + return model def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): @@ -144,8 +144,8 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): if heavy_budget > 0: mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) # Default: No padding applied to input else: - mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + # Recent Mask ones = torch.ones_like(attn_weights, dtype=torch.bool) ones = torch.triu(ones, diagonal=-recent_budget) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py index 2045808acc2..28f108cb636 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py index e84b276381a..2125f89a2c3 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -64,8 +64,7 @@ def __init__( self.h2o_min_seqlen = h2o_min_seqlen def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory + """Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory storage as `fused_qkv` Args: @@ -80,8 +79,7 @@ def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Ten return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :] def _merge_heads(self, x: torch.Tensor) -> torch.Tensor: - """ - Merge heads together over the last dimension + """Merge heads together over the last dimension. Args: x (`torch.tensor`, *required*): [batch_size * num_heads, seq_length, head_dim] @@ -201,8 +199,7 @@ def forward( return outputs def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor: - """ - Dropout add function + """Dropout add function. Args: x (`torch.tensor`, *required*): diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py index dee856047e0..f7e9851e814 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -37,7 +37,7 @@ def __init__( self.hidden_size = config.hidden_size if self.hidden_size % self.num_attention_heads != 0: raise ValueError( - "The hidden size is not divisble by the number of attention heads! Make sure to update them" + "The hidden size is not divisible by the number of attention heads! Make sure to update them" ) self.head_size = self.hidden_size // self.num_attention_heads self.rotary_ndims = int(self.head_size * config.rotary_pct) @@ -123,9 +123,7 @@ def forward( @classmethod def _split_heads(cls, tensor, num_attention_heads, attn_head_size): - """ - Splits hidden dim into attn_head_size and num_attention_heads - """ + """Splits hidden dim into attn_head_size and num_attention_heads.""" # tensor: [bs, seq_len, hidden_size] new_shape = tensor.size()[:-1] + (num_attention_heads, attn_head_size) # -> [bs, seq_len, num_attention_heads, attn_head_size] @@ -136,9 +134,7 @@ def _split_heads(cls, tensor, num_attention_heads, attn_head_size): @classmethod def _merge_heads(cls, tensor, num_attention_heads, attn_head_size): - """ - Merges attn_head_size dim and num_attn_heads dim into hidden dim - """ + """Merges attn_head_size dim and num_attn_heads dim into hidden dim.""" # tensor [bs, num_attention_heads, seq_len, attn_head_size] tensor = tensor.permute(0, 2, 1, 3).contiguous() # -> [bs, seq_len, num_attention_heads, attn_head_size] diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index a7dc1b64309..29276b09271 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" PyTorch llama model.""" +"""PyTorch llama model.""" import math from typing import List, Optional, Tuple, Union @@ -33,11 +33,11 @@ logger = logging.getLogger(__name__) class H2OLlamaAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" + """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__( self, - model, + model, config: LlamaConfig, heavy_ratio, recent_ratio, diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index 0019c553da3..a98a18e1ef0 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -29,14 +29,15 @@ logger = logging.getLogger(__name__) class H2OMistralAttention(nn.Module): - """ - Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + """Multi-headed attention from 'Attention Is All You Need' paper. + + Modified to use sliding window attention: Longformer and "Generating Long Sequences with Sparse Transformers". """ def __init__( self, - model, + model, config, heavy_ratio, recent_ratio, @@ -140,7 +141,7 @@ def forward( ) attn_weights = attn_weights + attention_mask - + # get hh mask if q_len > self.h2o_min_seqlen: mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index 41f789db03b..04b6c1975e3 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" PyTorch OPT model.""" +"""PyTorch OPT model.""" from typing import List, Optional, Tuple, Union import torch @@ -23,7 +23,7 @@ from ..h2o import get_hh_mask class H2OOPTAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" + """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__( self, From 3fd73cb88975066b7b84629e6986abab049a197e Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 7 May 2024 04:26:29 -0400 Subject: [PATCH 08/62] update Signed-off-by: n1ck-guo --- .../text-generation/h2o/run_generation.py | 219 ++++++++++++++++++ .../modeling/kv_cache_compression/h2o.py | 13 +- .../models/modeling_bloom.py | 2 +- .../models/modeling_mixtral.py | 169 ++++++++++++++ .../models/modeling_opt.py | 2 +- 5 files changed, 402 insertions(+), 3 deletions(-) create mode 100644 examples/huggingface/pytorch/text-generation/h2o/run_generation.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py new file mode 100644 index 00000000000..7b41ee5a5e2 --- /dev/null +++ b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py @@ -0,0 +1,219 @@ +import argparse +import sys +sys.path.insert(0, '/home/hengguo/code/intel-extension-for-transformers') +import time +import json +import torch +from transformers import AutoConfig, AutoTokenizer, AutoModel, AutoModelForCausalLM +from transformers.utils import check_min_version + +parser = argparse.ArgumentParser() +parser.add_argument("--model", default=None) +parser.add_argument( + "--dataset", nargs="?", default="NeelNanda/pile-10k", const="NeelNanda/pile-10k" +) +parser.add_argument( + "--max_new_tokens", default=32, type=int, help="output max new tokens" +) +parser.add_argument("--output_dir", nargs="?", default="./saved_results") +parser.add_argument("--int8", action="store_true") +parser.add_argument( + "--int8_bf16_mixed", + action="store_true", + help="by default it is int8-fp32 mixed, to enable int8 mixed amp bf16 (work on platforms like SPR)", +) +parser.add_argument( + "--restore", + action="store_true", + help="restore ipex quantized model from output_dir/best_configure.json", +) +parser.add_argument( + "--peft_model_id", type=str, default=None, help="model_name_or_path of peft model" +) +parser.add_argument("--_commit_hash", default=None, type=str) +parser.add_argument("--trust_remote_code", action="store_true") +parser.add_argument("--use_neural_speed", action="store_true") +# ============Benchmark configs============== +parser.add_argument("--benchmark", action="store_true") +parser.add_argument("--iters", default=100, type=int, help="num iter") +parser.add_argument("--num_warmup", default=10, type=int, help="num warmup") +# ============Accuracy configs============== +parser.add_argument("--accuracy", action="store_true") +parser.add_argument("--batch_size", default=56, type=int, help="batch size num.") +parser.add_argument( + "--save_accuracy_path", default=None, help="Save accuracy results path." +) +parser.add_argument( + "--tasks", + nargs="+", + default=["lambada_openai"], + type=str, + help="tasks list for accuracy validation", +) +# ============MixedPrecision configs============== +parser.add_argument("--mixed_precision", action="store_true") + +# ============h2o configs============== +parser.add_argument('--enable_small_cache', action='store_true') +parser.add_argument("--heavy_ratio", type=float, default=0.1) +parser.add_argument("--recent_ratio", type=float, default=0.1) +parser.add_argument("--device", type=str, default='cpu') +parser.add_argument("--h2o_min_seqlen", type=int, default=0) + +args = parser.parse_args() +# transformers version >= 4.32.0 contained the mpt modeling definition. +# https://github.com/huggingface/transformers/blob/main/src/transformers/models/mpt/modeling_mpt.py +# 4.31.0 for ipex.optimize_transformers +check_min_version("4.35.2") +# get model config +if args.peft_model_id: + from peft import PeftConfig + + peft_config = PeftConfig.from_pretrained(args.peft_model_id) + if args.model is None: + args.model = peft_config.base_model_name_or_path + print("we will use peft base_model_name_or_path to get tokenizer.") + +config = AutoConfig.from_pretrained( + args.model, + torchscript=False, + use_cache=True, # to use kv cache. + trust_remote_code=args.trust_remote_code, + _commit_hash=args._commit_hash, +) + +# chatglm +if config.model_type == "chatglm": + AutoModelForCausalLM = AutoModel +# tokenizer +if config.model_type == "llama": + from transformers import LlamaTokenizer + + tokenizer = LlamaTokenizer.from_pretrained(args.model) +else: + tokenizer = AutoTokenizer.from_pretrained( + args.model, trust_remote_code=args.trust_remote_code + ) + +# use peft +args.model = args.peft_model_id if args.peft_model_id is not None else args.model + +# Generation +if args.use_neural_speed: + generate_kwargs = dict(do_sample=False, temperature=0.9, num_beams=1) +else: + generate_kwargs = dict(do_sample=False, temperature=0.9, num_beams=4) + +if 'cpu' in args.device: + device = args.device +else: + device = f"cuda:{args.device}" +user_model = AutoModelForCausalLM.from_pretrained(args.model, trust_remote_code=args.trust_remote_code) +user_model.to(device) + +# get optimized model +if args.enable_small_cache: + print('Enable Small Cache Size') + # checkpoint = copy.deepcopy(model.state_dict()) + # model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) + from intel_extension_for_transformers.transformers.modeling.kv_cache_compression import convert_model + user_model = convert_model(user_model, heavy_ratio=args.heavy_ratio, recent_ratio=args.recent_ratio, h2o_min_seqlen=args.h2o_min_seqlen) + print("converted model: ", user_model) + +# save model +if args.output_dir is not None: + tokenizer.save_pretrained(args.output_dir) + user_model.save_pretrained(args.output_dir) + +if args.benchmark: + user_model = ( + user_model.eval() if (not (args.int8 or args.int8_bf16_mixed) and hasattr(user_model, "eval")) else user_model + ) + prompt = "Once upon a time, there existed a little girl, who liked to have adventures. She wanted to go to places and meet new people, and have fun." + input_size = tokenizer(prompt, return_tensors="pt").input_ids.size(dim=1) + print("---- Prompt size:", input_size) + + # start + total_time = 0.0 + num_iter = args.iters + num_warmup = args.num_warmup + total_token_num = 0 + eos_token_id = tokenizer.eos_token_id + with torch.inference_mode(), torch.no_grad(): + for i in range(num_iter): + tic = time.time() + if hasattr(tokenizer, "build_chat_input"): + input_ids = tokenizer.build_chat_input(prompt)["input_ids"] + input_ids = input_ids.repeat(args.batch_size, 1) + eos_token_id = [ + tokenizer.eos_token_id, + tokenizer.get_command("<|user|>"), + tokenizer.get_command("<|observation|>"), + ] + elif hasattr(tokenizer, "build_prompt"): + build_prompt = tokenizer.build_prompt(prompt) + input_ids = tokenizer( + [build_prompt] * args.batch_size, return_tensors="pt" + ).input_ids + else: + input_ids = tokenizer( + [prompt] * args.batch_size, return_tensors="pt" + ).input_ids + gen_ids = user_model.generate( + input_ids, + max_new_tokens=args.max_new_tokens, + **generate_kwargs, + eos_token_id=eos_token_id + ) + gen_text = tokenizer.batch_decode(gen_ids, skip_special_tokens=True) + toc = time.time() + # please check the gen_ids if include input_ids. + input_tokens_num = input_ids.numel() + output_tokens_num = torch.tensor(gen_ids).numel() - input_tokens_num + print(gen_text, flush=True) + if i >= num_warmup: + total_time += toc - tic + total_token_num += output_tokens_num + + print("\n", "-" * 10, "Summary:", "-" * 10) + latency = total_time / total_token_num + print("Inference latency: %.3f sec." % latency) + throughput = total_token_num / total_time + print("Throughput: {} samples/sec".format(throughput)) + +if args.accuracy: + user_model = (user_model.eval() if (not (args.int8 or args.int8_bf16_mixed) and hasattr(user_model, "eval")) \ + else user_model) + args.model = (peft_config.base_model_name_or_path if args.peft_model_id else args.model) + from intel_extension_for_transformers.transformers.llm.evaluation.lm_eval import evaluate + pretrained = ',pretrained=' + args.model + args._commit_hash = "main" if args._commit_hash is None else args._commit_hash + eval_args = "tokenizer=" + args.model + ",dtype=float32" + ",_commit_hash=" + \ + args._commit_hash + ",trust_remote_code=" + str(args.trust_remote_code) + if args.use_neural_speed: + eval_args += pretrained + q_conf = user_model.config.quantization_config + if isinstance(q_conf, dict): + q_algo = q_conf.get("quant_method", None) + else: + q_algo = q_conf.quant_method.value + if q_algo.upper() in ["AWQ", "GPTQ", "AUTOROUND"]: + eval_args += ",use_gptq=True" + results = evaluate( + model="hf-causal", + model_args=eval_args, + user_model=user_model, + batch_size=args.batch_size, + tasks=args.tasks, + model_format="neural_speed" if args.use_neural_speed else "torch", + device=device + ) + dumped = json.dumps(results, indent=2) + if args.save_accuracy_path: + with open(args.save_accuracy_path, "w") as f: + f.write(dumped) + for task_name in args.tasks: + if task_name == "wikitext": + print("Accuracy for %s is: %s" % (task_name, results["results"][task_name]["word_perplexity"])) + else: + print("Accuracy for %s is: %s" % (task_name, results["results"][task_name]["acc"])) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 589abb8642c..84ae21a18ae 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -141,6 +141,7 @@ def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=No def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): heavy_budget = int(heavy_budget_ratio * attn_weights.shape[-1]) recent_budget = int(recent_budget_ratio * attn_weights.shape[-1]) + print(heavy_budget, recent_budget) if heavy_budget > 0: mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) # Default: No padding applied to input else: @@ -150,9 +151,13 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): ones = torch.ones_like(attn_weights, dtype=torch.bool) ones = torch.triu(ones, diagonal=-recent_budget) mask_bottom = torch.logical_or(mask_bottom, ones) - # Combine h2o+recent and apply casual mask mask_bottom = torch.tril(mask_bottom, diagonal=0) + + # ones = torch.ones_like(attn_weights, dtype=torch.bool) + # ones = torch.tril(ones, diagonal=recent_budget) + # ones = torch.triu(ones, diagonal=-recent_budget) + # mask_bottom = torch.logical_or(mask_bottom, ones) return mask_bottom class H2OKVCache: @@ -242,3 +247,9 @@ def _update_hh_score(self, attn_score_cache): def _clean_scores(self): self.hh_score = None + +if __name__ == "__main__": + a = torch.randn(1, 10,10) + print(a) + mask = get_hh_mask(0, 0.7, a) + print(mask) \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py index 2125f89a2c3..627239e9904 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -159,7 +159,7 @@ def forward( # get hh mask if q_length > self.h2o_min_seqlen: mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - attn_weights[~mask_bottom] = torch.min(attention_mask) + attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min attention_probs = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(input_dtype) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py new file mode 100644 index 00000000000..acad6e9cbba --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import logging +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from transformers.cache_utils import Cache +from transformers.models.mixtral.modeling_mixtral import apply_rotary_pos_emb, repeat_kv + +from ..h2o import get_hh_mask + +logger = logging.getLogger(__name__) + +class MixtralAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__( + self, + model, + config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + ): + super().__init__() + self.config = config + self.layer_idx = model.layer_idx + if self.layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + self.attention_dropout = config.attention_dropout + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = model.q_proj + self.k_proj = model.k_proj + self.v_proj = model.v_proj + self.o_proj = model.o_proj + + self.rotary_emb = model.rotary_emb + + # for h2o + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if "padding_mask" in kwargs: + logger.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + attn_weights = attn_weights + attention_mask + + # get hh mask + if q_len > self.h2o_min_seqlen: + mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index 04b6c1975e3..e794e7ee92e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -141,7 +141,7 @@ def forward( # get hh mask if tgt_len > self.h2o_min_seqlen: mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - attn_weights[~mask_bottom] = torch.min(attention_mask) + attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 if attn_weights.dtype == torch.float16: From a2d3ae03790f00fa4eef89ce517f5c25e3ba9eab Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 08:27:30 +0000 Subject: [PATCH 09/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../transformers/modeling/kv_cache_compression/h2o.py | 2 +- .../kv_cache_compression/models/modeling_mixtral.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 84ae21a18ae..38e87f6b159 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -252,4 +252,4 @@ def _clean_scores(self): a = torch.randn(1, 10,10) print(a) mask = get_hh_mask(0, 0.7, a) - print(mask) \ No newline at end of file + print(mask) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index acad6e9cbba..35c04379b20 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -29,14 +29,15 @@ logger = logging.getLogger(__name__) class MixtralAttention(nn.Module): - """ - Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + """Multi-headed attention from 'Attention Is All You Need' paper. + + Modified to use sliding window attention: Longformer and "Generating Long Sequences with Sparse Transformers". """ def __init__( self, - model, + model, config, heavy_ratio, recent_ratio, @@ -141,7 +142,7 @@ def forward( ) attn_weights = attn_weights + attention_mask - + # get hh mask if q_len > self.h2o_min_seqlen: mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) @@ -166,4 +167,4 @@ def forward( if not output_attentions: attn_weights = None - return attn_output, attn_weights, past_key_value \ No newline at end of file + return attn_output, attn_weights, past_key_value From a83e6d61bdb7bdac23aaa94f08ace0af768c339c Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 14 May 2024 04:38:38 -0400 Subject: [PATCH 10/62] real drop Signed-off-by: n1ck-guo --- .../text-generation/h2o/run_generation.py | 78 ++++++++----------- .../modeling/kv_cache_compression/h2o.py | 33 ++++---- .../models/modeling_llama.py | 4 +- .../models/modeling_opt.py | 2 + 4 files changed, 54 insertions(+), 63 deletions(-) diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py index 7b41ee5a5e2..e354909c690 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py @@ -39,17 +39,16 @@ parser.add_argument("--num_warmup", default=10, type=int, help="num warmup") # ============Accuracy configs============== parser.add_argument("--accuracy", action="store_true") -parser.add_argument("--batch_size", default=56, type=int, help="batch size num.") +parser.add_argument("--batch_size", default=16, type=int, help="batch size num.") parser.add_argument( "--save_accuracy_path", default=None, help="Save accuracy results path." ) -parser.add_argument( - "--tasks", - nargs="+", - default=["lambada_openai"], - type=str, - help="tasks list for accuracy validation", -) +parser.add_argument("--output_excel", default=None, type=str) +parser.add_argument("--eval_bs", default=4, type=int, + help="eval batch size") +parser.add_argument("--tasks", nargs='+', default=["winogrande", "copa", "piqa", "rte", "hellaswag", \ + "openbookqa", "lambada_openai", "lambada_standard", "wikitext"], type=str, \ + help="tasks list for accuracy validation") # ============MixedPrecision configs============== parser.add_argument("--mixed_precision", action="store_true") @@ -89,7 +88,8 @@ if config.model_type == "llama": from transformers import LlamaTokenizer - tokenizer = LlamaTokenizer.from_pretrained(args.model) + # tokenizer = LlamaTokenizer.from_pretrained(args.model) + tokenizer = AutoTokenizer.from_pretrained(args.model) else: tokenizer = AutoTokenizer.from_pretrained( args.model, trust_remote_code=args.trust_remote_code @@ -121,9 +121,9 @@ print("converted model: ", user_model) # save model -if args.output_dir is not None: - tokenizer.save_pretrained(args.output_dir) - user_model.save_pretrained(args.output_dir) +# if args.output_dir is not None: +# tokenizer.save_pretrained(args.output_dir) +# user_model.save_pretrained(args.output_dir) if args.benchmark: user_model = ( @@ -184,36 +184,24 @@ if args.accuracy: user_model = (user_model.eval() if (not (args.int8 or args.int8_bf16_mixed) and hasattr(user_model, "eval")) \ else user_model) - args.model = (peft_config.base_model_name_or_path if args.peft_model_id else args.model) - from intel_extension_for_transformers.transformers.llm.evaluation.lm_eval import evaluate - pretrained = ',pretrained=' + args.model - args._commit_hash = "main" if args._commit_hash is None else args._commit_hash - eval_args = "tokenizer=" + args.model + ",dtype=float32" + ",_commit_hash=" + \ - args._commit_hash + ",trust_remote_code=" + str(args.trust_remote_code) - if args.use_neural_speed: - eval_args += pretrained - q_conf = user_model.config.quantization_config - if isinstance(q_conf, dict): - q_algo = q_conf.get("quant_method", None) - else: - q_algo = q_conf.quant_method.value - if q_algo.upper() in ["AWQ", "GPTQ", "AUTOROUND"]: - eval_args += ",use_gptq=True" - results = evaluate( - model="hf-causal", - model_args=eval_args, - user_model=user_model, - batch_size=args.batch_size, - tasks=args.tasks, - model_format="neural_speed" if args.use_neural_speed else "torch", - device=device - ) - dumped = json.dumps(results, indent=2) - if args.save_accuracy_path: - with open(args.save_accuracy_path, "w") as f: - f.write(dumped) - for task_name in args.tasks: - if task_name == "wikitext": - print("Accuracy for %s is: %s" % (task_name, results["results"][task_name]["word_perplexity"])) - else: - print("Accuracy for %s is: %s" % (task_name, results["results"][task_name]["acc"])) + from intel_extension_for_transformers.transformers.llm.evaluation.lm_eval import evaluate, LMEvalParser + model_args="pretrained="+args.model+",trust_remote_code="+str(args.trust_remote_code) + args.tasks = ",".join(args.tasks) + eval_args = LMEvalParser(model = "hf", + user_model=user_model, + tokenizer=tokenizer, + model_args=model_args, + tasks = args.tasks, + device = device, + output_path=args.save_accuracy_path, + batch_size = args.batch_size) + results = evaluate(eval_args) + # dumped = json.dumps(results, indent=2) + # if args.save_accuracy_path: + # with open(args.save_accuracy_path, "w") as f: + # f.write(dumped) + # for task_name in args.tasks.split(","): + # if task_name == "wikitext": + # print("Perplexity for %s is: %s" % (task_name, results["results"][task_name]["word_perplexity"])) + # else: + # print("Accuracy for %s is: %s" % (task_name, results["results"][task_name]["acc"])) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 38e87f6b159..a5b6fb5d6a1 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -67,7 +67,7 @@ def get_module(model, op_name): return module -def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024): +def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024, real_drop=True): device = model.device model_type = model.config.model_type model_name = model_type.replace("-", "_") @@ -80,7 +80,7 @@ def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024): for layer_name in atten_layers: module = get_module(model, layer_name) - module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen) + module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen, real_drop) set_module(model, layer_name, module) model = model.to(device) return model @@ -141,7 +141,6 @@ def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=No def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): heavy_budget = int(heavy_budget_ratio * attn_weights.shape[-1]) recent_budget = int(recent_budget_ratio * attn_weights.shape[-1]) - print(heavy_budget, recent_budget) if heavy_budget > 0: mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) # Default: No padding applied to input else: @@ -163,21 +162,21 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): class H2OKVCache: def __init__( self, - hh_size=4, - recent_size=512, + heavy_ratio=0.2, + recent_ratio=0.2, k_seq_dim=2, v_seq_dim=2, ): - self.hh_size = hh_size - self.recent_size = recent_size - self.cache_size = hh_size + recent_size + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio self.k_seq_dim = k_seq_dim self.v_seq_dim = v_seq_dim self.hh_score = None def __call__(self, past_key_values, attn_score_cache): - self._update_hh_score(attn_score_cache) + heavy_budget = int(self.heavy_budget_ratio * self.hh_score.shape[-1]) + recent_budget = int(self.recent_budget_ratio * self.hh_score.shape[-1]) if past_key_values is None: return None @@ -188,12 +187,12 @@ def __call__(self, past_key_values, attn_score_cache): # hh-selection bsz, num_heads, _, head_dim = past_key_values[0].shape - select_hh_scores = self.hh_score[:, :seq_len - self.recent_size] - _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) + select_hh_scores = self.hh_score[:, :seq_len - recent_budget] + _, keep_topk = torch.topk(select_hh_scores, heavy_budget, dim=-1) keep_topk = keep_topk.sort().values # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) - keep_recent = torch.arange(seq_len - self.recent_size, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) + keep_recent = torch.arange(seq_len - recent_budget, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) @@ -207,6 +206,8 @@ def __call__(self, past_key_values, attn_score_cache): return (k_hh_recent, v_hh_recent) def evict_for_space(self, past_key_values, num_coming): + heavy_budget = int(self.heavy_budget_ratio * num_coming.shape[-1]) + recent_budget = int(self.recent_budget_ratio * num_coming.shape[-1]) if past_key_values is None: return None seq_len = past_key_values[0][0].size(self.k_seq_dim) @@ -216,12 +217,12 @@ def evict_for_space(self, past_key_values, num_coming): # hh-selection bsz, num_heads, _, head_dim = past_key_values[0].shape - select_hh_scores = self.hh_score[:, :seq_len - self.recent_size + num_coming] - _, keep_topk = torch.topk(select_hh_scores, self.hh_size, dim=-1) + select_hh_scores = self.hh_score[:, :seq_len - recent_budget + num_coming] + _, keep_topk = torch.topk(select_hh_scores, heavy_budget, dim=-1) keep_topk = keep_topk.sort().values # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) - keep_recent = torch.arange(seq_len - self.recent_size + num_coming, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) + keep_recent = torch.arange(seq_len - recent_budget + num_coming, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) @@ -236,7 +237,7 @@ def evict_for_space(self, past_key_values, num_coming): def _update_hh_score(self, attn_score_cache): - num_new_tokens = attn_score_cache.shape[2] + num_new_tokens = attn_score_cache.shape[-2] if self.hh_score is None: self.hh_score = attn_score_cache.sum(0).sum(1) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 29276b09271..73825225c8a 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -83,7 +83,7 @@ def __init__( self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen - self.h2o_kv_cache = H2OKVCache() + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio) def forward( self, @@ -147,7 +147,7 @@ def forward( # get hh mask if q_len > self.h2o_min_seqlen: if self.real_drop: - pass + past_key_value = self.h2o_kv_cache(past_key_value, attn_weights.detach().clone()) else: mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index e794e7ee92e..bdec63f36ec 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -32,6 +32,7 @@ def __init__( heavy_ratio, recent_ratio, h2o_min_seqlen=1024, + real_drop=False ): super().__init__() self.config = config @@ -60,6 +61,7 @@ def __init__( self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen + self.real_drop = real_drop def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() From 92c8a62d72acbbd87d16c53ba730bb2352c2c40f Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Wed, 15 May 2024 05:11:28 -0400 Subject: [PATCH 11/62] modify real drop code Signed-off-by: n1ck-guo --- .../text-generation/h2o/run_generation.py | 17 +-- .../modeling/kv_cache_compression/h2o.py | 127 +++++++++--------- .../models/modeling_llama.py | 24 ++-- 3 files changed, 81 insertions(+), 87 deletions(-) diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py index e354909c690..6fcef7e5c57 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py @@ -49,11 +49,13 @@ parser.add_argument("--tasks", nargs='+', default=["winogrande", "copa", "piqa", "rte", "hellaswag", \ "openbookqa", "lambada_openai", "lambada_standard", "wikitext"], type=str, \ help="tasks list for accuracy validation") +parser.add_argument("--num_fewshot", default=0, type=int, help="num few shot.") # ============MixedPrecision configs============== parser.add_argument("--mixed_precision", action="store_true") # ============h2o configs============== parser.add_argument('--enable_small_cache', action='store_true') +parser.add_argument('--real_drop', action='store_true') parser.add_argument("--heavy_ratio", type=float, default=0.1) parser.add_argument("--recent_ratio", type=float, default=0.1) parser.add_argument("--device", type=str, default='cpu') @@ -117,7 +119,7 @@ # checkpoint = copy.deepcopy(model.state_dict()) # model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) from intel_extension_for_transformers.transformers.modeling.kv_cache_compression import convert_model - user_model = convert_model(user_model, heavy_ratio=args.heavy_ratio, recent_ratio=args.recent_ratio, h2o_min_seqlen=args.h2o_min_seqlen) + user_model = convert_model(user_model, heavy_ratio=args.heavy_ratio, recent_ratio=args.recent_ratio, h2o_min_seqlen=args.h2o_min_seqlen, real_drop=args.read_drop) print("converted model: ", user_model) # save model @@ -193,15 +195,8 @@ model_args=model_args, tasks = args.tasks, device = device, + num_fewshot=args.num_fewshot, output_path=args.save_accuracy_path, batch_size = args.batch_size) - results = evaluate(eval_args) - # dumped = json.dumps(results, indent=2) - # if args.save_accuracy_path: - # with open(args.save_accuracy_path, "w") as f: - # f.write(dumped) - # for task_name in args.tasks.split(","): - # if task_name == "wikitext": - # print("Perplexity for %s is: %s" % (task_name, results["results"][task_name]["word_perplexity"])) - # else: - # print("Accuracy for %s is: %s" % (task_name, results["results"][task_name]["acc"])) + print("using device:", device) + results = evaluate(eval_args) \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index a5b6fb5d6a1..469c5f2ed31 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -14,6 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import math from collections import OrderedDict import importlib @@ -159,98 +160,90 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): # mask_bottom = torch.logical_or(mask_bottom, ones) return mask_bottom +def _get_attn_weights(query_states, key_states, value_states, **kwargs): + head_dim = query_states.size(-1) + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(head_dim) + if "attention_mask" in kwargs: + attention_mask = kwargs["attention_mask"] + if attention_mask is not None: + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + return attn_weights + class H2OKVCache: def __init__( self, heavy_ratio=0.2, recent_ratio=0.2, - k_seq_dim=2, - v_seq_dim=2, ): + ## bsz, num_heads, seq_len, head_dim | num_heads, seq_len, head_dim self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio - self.k_seq_dim = k_seq_dim - self.v_seq_dim = v_seq_dim self.hh_score = None - - def __call__(self, past_key_values, attn_score_cache): - self._update_hh_score(attn_score_cache) - heavy_budget = int(self.heavy_budget_ratio * self.hh_score.shape[-1]) - recent_budget = int(self.recent_budget_ratio * self.hh_score.shape[-1]) - - if past_key_values is None: - return None - seq_len = past_key_values[0].size(self.k_seq_dim) - if seq_len <= self.cache_size: - return past_key_values - - # hh-selection - bsz, num_heads, _, head_dim = past_key_values[0].shape - - select_hh_scores = self.hh_score[:, :seq_len - recent_budget] - _, keep_topk = torch.topk(select_hh_scores, heavy_budget, dim=-1) - keep_topk = keep_topk.sort().values - - # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) - keep_recent = torch.arange(seq_len - recent_budget, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) - keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) - - mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) - mask = mask.scatter(-1, keep_idx, 1) - - k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) - v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) - - self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) - - return (k_hh_recent, v_hh_recent) - - def evict_for_space(self, past_key_values, num_coming): - heavy_budget = int(self.heavy_budget_ratio * num_coming.shape[-1]) - recent_budget = int(self.recent_budget_ratio * num_coming.shape[-1]) - if past_key_values is None: - return None - seq_len = past_key_values[0][0].size(self.k_seq_dim) - if seq_len + num_coming <= self.cache_size: - return past_key_values + self.score_func = _get_attn_weights + self.idx = 0 + + + def __call__(self, query_states, key_states, value_states, **kwargs): + self.idx += 1 + attn_score = self.score_func(query_states, key_states, value_states, **kwargs) + self._update_hh_score(attn_score, mean=False) + heavy_budget = int(self.heavy_ratio * self.hh_score.shape[-1]) + recent_budget = int(self.recent_ratio * self.hh_score.shape[-1]) + cache_size = heavy_budget + recent_budget + + seq_len = key_states.size(-2) + if seq_len <= cache_size: + return key_states, value_states # hh-selection - bsz, num_heads, _, head_dim = past_key_values[0].shape + if len(self.hh_score) == 2: + select_hh_scores = self.hh_score[:, :seq_len - recent_budget] + else: + select_hh_scores = self.hh_score[:, :, :seq_len - recent_budget] - select_hh_scores = self.hh_score[:, :seq_len - recent_budget + num_coming] _, keep_topk = torch.topk(select_hh_scores, heavy_budget, dim=-1) keep_topk = keep_topk.sort().values # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) - keep_recent = torch.arange(seq_len - recent_budget + num_coming, seq_len, device=keep_topk.device).repeat(keep_topk.shape[0], 1) + repeat_shape = list(keep_topk.shape) + repeat_shape[-1] = 1 + keep_recent = torch.arange(seq_len - recent_budget, seq_len, device=keep_topk.device).repeat(repeat_shape) keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) - mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(past_key_values[0].device) + mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(key_states.device) mask = mask.scatter(-1, keep_idx, 1) - k_hh_recent = past_key_values[0].squeeze()[mask].view(bsz, num_heads, -1, head_dim) - v_hh_recent = past_key_values[1].squeeze()[mask].view(bsz, num_heads, -1, head_dim) + states_shape = list(key_states.shape) + states_shape[-2] = cache_size + k_hh_recent = key_states[mask].view(*states_shape) + states_shape[-1] = -1 + v_hh_recent = value_states[mask].view(*states_shape) - self.hh_score= self.hh_score[mask].view(num_heads, self.cache_size) + tmp_shape = list(self.hh_score.shape) + tmp_shape[-1] = cache_size + self.hh_score = self.hh_score[mask].view(*tmp_shape) + return k_hh_recent, v_hh_recent - return (k_hh_recent, v_hh_recent) + def _update_hh_score(self, attn_score_cache, mean=False): + # hh_score size (bsz, num_heads, seq_len) or (num_heads, seq_len) + num_new_tokens = attn_score_cache.shape[-2] - def _update_hh_score(self, attn_score_cache): + attn_score_cache = attn_score_cache.sum(-2) + if self.hh_score is not None: + if len(attn_score_cache) == 3: + attn_score_cache[:, :self.hh_score.shape[-1]] += self.hh_score / (1 if not mean else self.idx) + else: + attn_score_cache[:, :, :self.hh_score.shape[-1]] += self.hh_score / (1 if not mean else self.idx) - num_new_tokens = attn_score_cache.shape[-2] + self.hh_score = attn_score_cache - if self.hh_score is None: - self.hh_score = attn_score_cache.sum(0).sum(1) - else: - attn_score_cache = attn_score_cache.sum(0).sum(1) - attn_score_cache[:, :-num_new_tokens] += self.hh_score - self.hh_score = attn_score_cache + # if self.hh_score is None: + # self.hh_score = attn_score_cache.sum(-2) + # else: + # attn_score_cache = attn_score_cache.sum(-2) + # attn_score_cache[:, :-num_new_tokens] += self.hh_score + # self.hh_score = attn_score_cache def _clean_scores(self): self.hh_score = None - -if __name__ == "__main__": - a = torch.randn(1, 10,10) - print(a) - mask = get_hh_mask(0, 0.7, a) - print(mask) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 73825225c8a..ffc360e660a 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -90,14 +90,13 @@ def forward( hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, + past_key_value: Optional[Cache] = None, # transformers.cache_utils.DynamicCache output_attentions: bool = False, use_cache: bool = False, cache_position: Optional[torch.LongTensor] = None, **kwargs, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: bsz, q_len, _ = hidden_states.size() - if self.config.pretraining_tp > 1: key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp query_slices = self.q_proj.weight.split( @@ -132,22 +131,29 @@ def forward( # sin and cos are specific to RoPE models; cache_position needed for the static cache cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) if attention_mask is not None: # no matter the length, we just slice it - causal_mask = attention_mask - if cache_position is not None: - causal_mask = attention_mask[:, :, cache_position, : key_states.shape[-2]] + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] attn_weights = attn_weights + causal_mask - # get hh mask + # H2O if q_len > self.h2o_min_seqlen: - if self.real_drop: - past_key_value = self.h2o_kv_cache(past_key_value, attn_weights.detach().clone()) + if self.real_drop and past_key_value is not None: + if len(past_key_value.key_cache) == self.layer_idx + 1: + self.h2o_kv_cache._clean_scores() + new_key_states, new_value_states = self.h2o_kv_cache( + query_states, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + attention_mask=attention_mask + ) + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states else: mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min From 70a1cf3fc0fa47661755f8d79ad6d469a1e326f3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 09:12:34 +0000 Subject: [PATCH 12/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../transformers/modeling/kv_cache_compression/h2o.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 469c5f2ed31..4191cb83f40 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -180,9 +180,9 @@ def __init__( self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.hh_score = None - self.score_func = _get_attn_weights + self.score_func = _get_attn_weights self.idx = 0 - + def __call__(self, query_states, key_states, value_states, **kwargs): self.idx += 1 From bc9eadea0fc2f54d53a1da23de3441a38dea3e75 Mon Sep 17 00:00:00 2001 From: "biao.fang" Date: Thu, 16 May 2024 10:38:39 +0800 Subject: [PATCH 13/62] fix Signed-off-by: biao.fang --- .../modeling/kv_cache_compression/h2o.py | 40 +++++++------------ .../models/modeling_llama.py | 8 ++-- .../models/modeling_opt.py | 36 ++++++++++++++--- 3 files changed, 50 insertions(+), 34 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 4191cb83f40..f8abcf82655 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -68,7 +68,7 @@ def get_module(model, op_name): return module -def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024, real_drop=True): +def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024, real_drop=True, is_gen=False): device = model.device model_type = model.config.model_type model_name = model_type.replace("-", "_") @@ -81,7 +81,7 @@ def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024, real_dr for layer_name in atten_layers: module = get_module(model, layer_name) - module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen, real_drop) + module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen, real_drop, is_gen) set_module(model, layer_name, module) model = model.to(device) return model @@ -160,34 +160,24 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): # mask_bottom = torch.logical_or(mask_bottom, ones) return mask_bottom -def _get_attn_weights(query_states, key_states, value_states, **kwargs): - head_dim = query_states.size(-1) - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(head_dim) - if "attention_mask" in kwargs: - attention_mask = kwargs["attention_mask"] - if attention_mask is not None: - causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] - attn_weights = attn_weights + causal_mask - return attn_weights - class H2OKVCache: def __init__( self, heavy_ratio=0.2, recent_ratio=0.2, + real_drop=True ): ## bsz, num_heads, seq_len, head_dim | num_heads, seq_len, head_dim self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.hh_score = None - self.score_func = _get_attn_weights + self.real_drop = real_drop self.idx = 0 - def __call__(self, query_states, key_states, value_states, **kwargs): + def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): self.idx += 1 - attn_score = self.score_func(query_states, key_states, value_states, **kwargs) - self._update_hh_score(attn_score, mean=False) + self._update_hh_score(attn_score, mean=mean) heavy_budget = int(self.heavy_ratio * self.hh_score.shape[-1]) recent_budget = int(self.recent_ratio * self.hh_score.shape[-1]) cache_size = heavy_budget + recent_budget @@ -197,7 +187,7 @@ def __call__(self, query_states, key_states, value_states, **kwargs): return key_states, value_states # hh-selection - if len(self.hh_score) == 2: + if len(self.hh_score.shape) == 2: select_hh_scores = self.hh_score[:, :seq_len - recent_budget] else: select_hh_scores = self.hh_score[:, :, :seq_len - recent_budget] @@ -213,6 +203,8 @@ def __call__(self, query_states, key_states, value_states, **kwargs): mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(key_states.device) mask = mask.scatter(-1, keep_idx, 1) + if not self.real_drop: + return mask states_shape = list(key_states.shape) states_shape[-2] = cache_size @@ -227,23 +219,19 @@ def __call__(self, query_states, key_states, value_states, **kwargs): def _update_hh_score(self, attn_score_cache, mean=False): # hh_score size (bsz, num_heads, seq_len) or (num_heads, seq_len) - num_new_tokens = attn_score_cache.shape[-2] attn_score_cache = attn_score_cache.sum(-2) if self.hh_score is not None: - if len(attn_score_cache) == 3: + # clean self.hh_score if not generation mode + if attn_score_cache.shape[-1] < self.hh_score: + self.clean_scores() + elif len(attn_score_cache.shape) == 2: attn_score_cache[:, :self.hh_score.shape[-1]] += self.hh_score / (1 if not mean else self.idx) else: attn_score_cache[:, :, :self.hh_score.shape[-1]] += self.hh_score / (1 if not mean else self.idx) self.hh_score = attn_score_cache - # if self.hh_score is None: - # self.hh_score = attn_score_cache.sum(-2) - # else: - # attn_score_cache = attn_score_cache.sum(-2) - # attn_score_cache[:, :-num_new_tokens] += self.hh_score - # self.hh_score = attn_score_cache - def _clean_scores(self): + def clean_scores(self): self.hh_score = None diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index ffc360e660a..fe6a3d2cf0c 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -42,7 +42,8 @@ def __init__( heavy_ratio, recent_ratio, h2o_min_seqlen=1024, - real_drop=False + real_drop=False, + is_gen=False ): super().__init__() self.config = config @@ -78,6 +79,7 @@ def __init__( # for h2o self.real_drop = real_drop + self.is_gen = is_gen self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio @@ -143,8 +145,8 @@ def forward( # H2O if q_len > self.h2o_min_seqlen: if self.real_drop and past_key_value is not None: - if len(past_key_value.key_cache) == self.layer_idx + 1: - self.h2o_kv_cache._clean_scores() + if not self.is_gen: + self.h2o_kv_cache.clean_scores() new_key_states, new_value_states = self.h2o_kv_cache( query_states, past_key_value.key_cache[self.layer_idx], diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index bdec63f36ec..da4ace562eb 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -20,7 +20,7 @@ import torch import torch.nn as nn -from ..h2o import get_hh_mask +from ..h2o import get_hh_mask, H2OKVCache class H2OOPTAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper.""" @@ -32,7 +32,8 @@ def __init__( heavy_ratio, recent_ratio, h2o_min_seqlen=1024, - real_drop=False + real_drop=False, + is_gen=False ): super().__init__() self.config = config @@ -58,10 +59,14 @@ def __init__( self.out_proj = model.out_proj # for h2o + self.is_gen = is_gen + self.real_drop = real_drop + self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen - self.real_drop = real_drop + + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -142,8 +147,29 @@ def forward( # get hh mask if tgt_len > self.h2o_min_seqlen: - mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask + ) + past_key_value = (new_key_states, new_value_states) + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask + ) + mask = mask.unsqueeze(1) + attn_weights = attn_weights * mask + ~mask * torch.finfo(attn_weights.dtype).min + + # sim + # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 if attn_weights.dtype == torch.float16: From 9aa25f65aa287a10de4e5606ab33d1a9bb78459a Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Thu, 16 May 2024 02:34:01 -0400 Subject: [PATCH 14/62] update for real drop and sim mode, using the same api Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/h2o.py | 21 ++++---- .../models/modeling_llama.py | 24 ++++++--- .../models/modeling_opt.py | 51 +++++++++---------- 3 files changed, 51 insertions(+), 45 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index f8abcf82655..afef8f333d1 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -187,25 +187,24 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): return key_states, value_states # hh-selection + mask = torch.zeros(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) if len(self.hh_score.shape) == 2: + if not recent_budget == 0: + mask[:,-recent_budget:] = 1 select_hh_scores = self.hh_score[:, :seq_len - recent_budget] else: - select_hh_scores = self.hh_score[:, :, :seq_len - recent_budget] + if not recent_budget == 0: + mask[:,:,-recent_budget:] = 1 + select_hh_scores = self.hh_score[:,:,:seq_len - recent_budget] - _, keep_topk = torch.topk(select_hh_scores, heavy_budget, dim=-1) - keep_topk = keep_topk.sort().values + if not heavy_budget == 0: + _, keep_topk = torch.topk(select_hh_scores, heavy_budget, dim=-1, largest=True) + mask = mask.scatter(-1, keep_topk, 1) - # keep_recent = torch.arange(seq_len - self.recent_size, seq_len).expand(keep_topk.shape[0], 1).to(keep_topk.device) - repeat_shape = list(keep_topk.shape) - repeat_shape[-1] = 1 - keep_recent = torch.arange(seq_len - recent_budget, seq_len, device=keep_topk.device).repeat(repeat_shape) - keep_idx = torch.cat([keep_topk, keep_recent], dim=-1) - - mask = torch.zeros(self.hh_score.shape, dtype=torch.bool).to(key_states.device) - mask = mask.scatter(-1, keep_idx, 1) if not self.real_drop: return mask + mask = mask.bool() states_shape = list(key_states.shape) states_shape[-2] = cache_size k_hh_recent = key_states[mask].view(*states_shape) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index fe6a3d2cf0c..1679425f92c 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -142,27 +142,35 @@ def forward( causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] attn_weights = attn_weights + causal_mask + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + # H2O if q_len > self.h2o_min_seqlen: + if not self.is_gen: + self.h2o_kv_cache.clean_scores() if self.real_drop and past_key_value is not None: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() new_key_states, new_value_states = self.h2o_kv_cache( query_states, past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], attention_mask=attention_mask ) - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states else: - mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + mask = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask + ) + attn_weights = attn_weights * mask.unsqueeze(2) + value_states = value_states * mask.unsqueeze(-1) + # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index da4ace562eb..d9367619c89 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -145,32 +145,6 @@ def forward( ) attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - # get hh mask - if tgt_len > self.h2o_min_seqlen: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - attention_mask=attention_mask - ) - past_key_value = (new_key_states, new_value_states) - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - attention_mask=attention_mask - ) - mask = mask.unsqueeze(1) - attn_weights = attn_weights * mask + ~mask * torch.finfo(attn_weights.dtype).min - - # sim - # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min - # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 if attn_weights.dtype == torch.float16: attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(torch.float16) @@ -195,6 +169,31 @@ def forward( attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) else: attn_weights_reshaped = None + + # get hh mask + if tgt_len > self.h2o_min_seqlen: + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask + ) + past_key_value = (new_key_states, new_value_states) + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask + ) + attn_weights = attn_weights * mask.unsqueeze(1) + + # sim + # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) From e51b5b961fae368c0b0ed046ab0caa52630bb70d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 06:35:57 +0000 Subject: [PATCH 15/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../transformers/modeling/kv_cache_compression/h2o.py | 4 ++-- .../modeling/kv_cache_compression/models/modeling_opt.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index afef8f333d1..bf9feb0a723 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -190,11 +190,11 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): mask = torch.zeros(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) if len(self.hh_score.shape) == 2: if not recent_budget == 0: - mask[:,-recent_budget:] = 1 + mask[:,-recent_budget:] = 1 select_hh_scores = self.hh_score[:, :seq_len - recent_budget] else: if not recent_budget == 0: - mask[:,:,-recent_budget:] = 1 + mask[:,:,-recent_budget:] = 1 select_hh_scores = self.hh_score[:,:,:seq_len - recent_budget] if not heavy_budget == 0: diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index d9367619c89..22ed38a512e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -169,7 +169,7 @@ def forward( attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) else: attn_weights_reshaped = None - + # get hh mask if tgt_len > self.h2o_min_seqlen: if not self.is_gen: From b8e9df288667b22214de373af96e836f37d630b0 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Thu, 16 May 2024 04:58:06 -0400 Subject: [PATCH 16/62] support for sdpa and flash attention Signed-off-by: n1ck-guo --- .../text-generation/h2o/run_generation.py | 14 +- .../modeling/kv_cache_compression/h2o.py | 46 +- .../models/modeling_llama.py | 408 ++++++++++++++++-- .../models/modeling_opt.py | 230 +++++++++- 4 files changed, 628 insertions(+), 70 deletions(-) diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py index 6fcef7e5c57..cf10e792e2e 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py @@ -54,7 +54,8 @@ parser.add_argument("--mixed_precision", action="store_true") # ============h2o configs============== -parser.add_argument('--enable_small_cache', action='store_true') +parser.add_argument('--h2o', action='store_true') +parser.add_argument('--is_gen', action='store_true') parser.add_argument('--real_drop', action='store_true') parser.add_argument("--heavy_ratio", type=float, default=0.1) parser.add_argument("--recent_ratio", type=float, default=0.1) @@ -114,12 +115,19 @@ user_model.to(device) # get optimized model -if args.enable_small_cache: +if args.h2o: print('Enable Small Cache Size') # checkpoint = copy.deepcopy(model.state_dict()) # model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) from intel_extension_for_transformers.transformers.modeling.kv_cache_compression import convert_model - user_model = convert_model(user_model, heavy_ratio=args.heavy_ratio, recent_ratio=args.recent_ratio, h2o_min_seqlen=args.h2o_min_seqlen, real_drop=args.read_drop) + user_model = convert_model( + user_model, + heavy_ratio=args.heavy_ratio, + recent_ratio=args.recent_ratio, + h2o_min_seqlen=args.h2o_min_seqlen, + real_drop=args.real_drop, + is_gen=args.is_gen + ) print("converted model: ", user_model) # save model diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index afef8f333d1..ae5c568788d 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -21,16 +21,6 @@ import torch from torch import nn -ATTENTION_MAPPING_NAMES = OrderedDict( - [ - ("opt", "OPTAttention"), - ("llama", "LlamaAttention"), - ("gpt_neox", "GPTNeoXAttention"), - ("mistral", "MistralAttention"), - ("bloom", "BloomAttention"), - ] -) - def set_module(model, op_name, new_module): """Set module with a given op name. @@ -67,21 +57,30 @@ def get_module(model, op_name): module = module return module - -def convert_model(model, heavy_ratio, recent_ratio, h2o_min_seqlen=1024, real_drop=True, is_gen=False): - device = model.device +def convert_model( + model, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + real_drop=True, + is_gen=False + ): model_type = model.config.model_type - model_name = model_type.replace("-", "_") - atten_cls = getattr(importlib.import_module(f".{model_name}.modeling_{model_name}", "transformers.models"), ATTENTION_MAPPING_NAMES[model_type]) - h2o_cls = getattr(importlib.import_module(f".models.modeling_{model_name}", "intel_extension_for_transformers.transformers.modeling.kv_cache_compression"), "H2O" + ATTENTION_MAPPING_NAMES[model_type]) + device = model.device atten_layers = [] for name, module in model.named_modules(): - if isinstance(module, atten_cls): + if "Attention" in module.__class__.__name__: atten_layers.append(name) - + for layer_name in atten_layers: module = get_module(model, layer_name) - module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen, real_drop, is_gen) + h2o_cls = getattr( + importlib.import_module( + f".models.modeling_{model_type}", + "intel_extension_for_transformers.transformers.modeling.kv_cache_compression" + ), + "H2O" + module.__class__.__name__) + module = h2o_cls(module, module.config, heavy_ratio, recent_ratio, h2o_min_seqlen, real_drop, is_gen) set_module(model, layer_name, module) model = model.to(device) return model @@ -165,17 +164,24 @@ def __init__( self, heavy_ratio=0.2, recent_ratio=0.2, - real_drop=True + real_drop=False, + min_seqlen=-1 ): ## bsz, num_heads, seq_len, head_dim | num_heads, seq_len, head_dim self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.hh_score = None self.real_drop = real_drop + self.min_seqlen = min_seqlen self.idx = 0 def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): + if key_states.shape[-2] <= self.min_seqlen: + if self.real_drop: + return key_states, value_states + else: + return torch.ones(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) self.idx += 1 self._update_hh_score(attn_score, mean=mean) heavy_budget = int(self.heavy_ratio * self.hh_score.shape[-1]) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 1679425f92c..2feaf1f8997 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -26,7 +26,12 @@ from transformers.cache_utils import Cache from transformers.models.llama.configuration_llama import LlamaConfig -from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv +from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv, _get_unpad_data +from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available + +if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa from ..h2o import get_hh_mask, H2OKVCache @@ -85,7 +90,7 @@ def __init__( self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio) + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) def forward( self, @@ -147,29 +152,30 @@ def forward( attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) # H2O - if q_len > self.h2o_min_seqlen: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop and past_key_value is not None: - new_key_states, new_value_states = self.h2o_kv_cache( - query_states, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - attention_mask=attention_mask - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - attention_mask=attention_mask + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop and past_key_value is not None: + new_key_states, new_value_states = self.h2o_kv_cache( + query_states, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + attention_mask=attention_mask, + mean=False ) - attn_weights = attn_weights * mask.unsqueeze(2) - value_states = value_states * mask.unsqueeze(-1) - # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask, + mean=False + ) + attn_weights = attn_weights * mask.unsqueeze(2) + value_states = value_states * mask.unsqueeze(-1) + # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min attn_output = torch.matmul(attn_weights, value_states) @@ -194,3 +200,357 @@ def forward( attn_weights = None return attn_output, attn_weights, past_key_value + +class H2OLlamaFlashAttention2(H2OLlamaAttention): + """ + Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + output_attentions = False + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dim x hidden_dim + # therefore we just need to keep the original shape + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + cos, sin = self.rotary_emb(value_states, position_ids) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + past_key_value = getattr(self, "past_key_value", past_key_value) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache + # to be able to avoid many of these transpose/reshape/view. + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + dropout_rate = self.attention_dropout if self.training else 0.0 + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in the correct dtype just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. (LlamaRMSNorm handles it correctly) + + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop and past_key_value is not None: + new_key_states, new_value_states = self.h2o_kv_cache( + query_states, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + attention_mask=attention_mask, + mean=False + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask, + mean=False + ) + key_states = key_states * mask.unsqueeze(-1) + value_states = value_states * mask.unsqueeze(-1) + + attn_output = self._flash_attention_forward( + query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate + ) + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + def _flash_attention_forward( + self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + """ + if not self._flash_attn_uses_top_left_mask: + causal = self.is_causal + else: + # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. + causal = self.is_causal and query_length != 1 + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + attn_output = flash_attn_func( + query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal + ) + + return attn_output + + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape + + key_layer = index_first_axis( + key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + value_layer = index_first_axis( + value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + +class H2OLlamaSdpaAttention(H2OLlamaAttention): + """ + Llama attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + `LlamaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from LlamaAttention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "LlamaModel is using LlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + ) + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + cos, sin = self.rotary_emb(value_states, position_ids) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + # In case static cache is used, it is an instance attribute. + past_key_value = getattr(self, "past_key_value", past_key_value) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + causal_mask = attention_mask + if attention_mask is not None: + causal_mask = causal_mask[:, :, :, : key_states.shape[-2]] + + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # Reference: https://github.com/pytorch/pytorch/issues/112577. + if query_states.device.type == "cuda" and causal_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + # h2o + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop and past_key_value is not None: + new_key_states, new_value_states = self.h2o_kv_cache( + query_states, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + attention_mask=attention_mask, + mean=False + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask, + mean=False + ) + key_states = key_states * mask.unsqueeze(-1) + value_states = value_states * mask.unsqueeze(-1) + + # In case we are not compiling, we may set `causal_mask` to None, which is required to dispatch to SDPA's Flash Attention 2 backend, rather + # relying on the `is_causal` argument. + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=causal_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + is_causal=causal_mask is None and q_len > 1, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value + +ATTENTION_CLASSES = { + "eager": H2OLlamaAttention, + "flash_attention_2": H2OLlamaFlashAttention2, + "sdpa": H2OLlamaSdpaAttention, +} \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index d9367619c89..305dda47efd 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -21,6 +21,9 @@ import torch.nn as nn from ..h2o import get_hh_mask, H2OKVCache +from transformers.utils import is_flash_attn_greater_or_equal_2_10, logging + +logger = logging.get_logger(__name__) class H2OOPTAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper.""" @@ -66,7 +69,7 @@ def __init__( self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop) + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -171,29 +174,30 @@ def forward( attn_weights_reshaped = None # get hh mask - if tgt_len > self.h2o_min_seqlen: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - attention_mask=attention_mask - ) - past_key_value = (new_key_states, new_value_states) - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - attention_mask=attention_mask - ) - attn_weights = attn_weights * mask.unsqueeze(1) + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask, + mean=False + ) + past_key_value = (new_key_states, new_value_states) + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask, + mean=False + ) + attn_weights = attn_weights * mask.unsqueeze(1) - # sim - # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + # sim + # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) @@ -215,3 +219,183 @@ def forward( attn_output = self.out_proj(attn_output) return attn_output, attn_weights_reshaped, past_key_value + + +class H2OOptFlashAttention2(H2OOPTAttention): + """ + OPT flash attention module. This module inherits from `OPTAttention` as the weights of the module stays untouched. + The only required change would be on the forward pass where it needs to correctly call the public API of flash + attention and deal with padding tokens in case the input contains any of them. + """ + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + + bsz, _, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) + # get key, value proj + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + query_length = query_states.shape[1] + tgt_len = key_states.shape[-2] + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dim x hidden_dim + query_states = query_states.view(bsz, query_length, self.num_heads, self.head_dim) + key_states = key_states.transpose(1, 2).view(bsz, tgt_len, self.num_heads, self.head_dim) + value_states = value_states.transpose(1, 2).view(bsz, tgt_len, self.num_heads, self.head_dim) + + attn_dropout = self.dropout if self.training else 0.0 + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 just to be sure everything works as expected. + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + # h2o + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = torch.max( + attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min, device=attn_weights.device) + ) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 + if attn_weights.dtype == torch.float16: + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(torch.float16) + else: + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if layer_head_mask is not None: + if layer_head_mask.size() != (self.num_heads,): + raise ValueError( + f"Head mask for a single layer should be of size {(self.num_heads,)}, but is" + f" {layer_head_mask.size()}" + ) + attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if output_attentions: + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to be reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask, + mean=False + ) + past_key_value = (new_key_states, new_value_states) + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + attention_mask=attention_mask, + mean=False + ) + key_states = key_states * mask.unsqueeze(-1) + value_states = value_states * mask.unsqueeze(-1) + + attn_output = self._flash_attention_forward( + query_states, key_states, value_states, attention_mask, query_length, dropout=attn_dropout + ) + + attn_weights_reshaped = attn_output.reshape(bsz, query_length, self.num_heads * self.head_dim) + attn_output = self.out_proj(attn_weights_reshaped) + + if not output_attentions: + attn_weights_reshaped = None + + return attn_output, attn_weights_reshaped, past_key_value \ No newline at end of file From 274b7ed03d802fe10a2173638e37caf9ba68b6fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 08:59:50 +0000 Subject: [PATCH 17/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../modeling/kv_cache_compression/h2o.py | 2 +- .../models/modeling_llama.py | 16 +++++++++------- .../kv_cache_compression/models/modeling_opt.py | 9 +++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index a5179a86f24..68d38c42780 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -71,7 +71,7 @@ def convert_model( for name, module in model.named_modules(): if "Attention" in module.__class__.__name__: atten_layers.append(name) - + for layer_name in atten_layers: module = get_module(model, layer_name) h2o_cls = getattr( diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 2feaf1f8997..7163e03c6a4 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -202,8 +202,9 @@ def forward( return attn_output, attn_weights, past_key_value class H2OLlamaFlashAttention2(H2OLlamaAttention): - """ - Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays + """Llama flash attention module. + + This module inherits from `LlamaAttention` as the weights of the module stays untouched. The only required change would be on the forward pass where it needs to correctly call the public API of flash attention and deal with padding tokens in case the input contains any of them. """ @@ -212,7 +213,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() @@ -285,7 +286,7 @@ def forward( query_states = query_states.to(target_dtype) key_states = key_states.to(target_dtype) value_states = value_states.to(target_dtype) - + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) if attention_mask is not None: # no matter the length, we just slice it @@ -429,8 +430,9 @@ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query ) class H2OLlamaSdpaAttention(H2OLlamaAttention): - """ - Llama attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + """Llama attention module using torch.nn.functional.scaled_dot_product_attention. + + This module inherits from `LlamaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to SDPA API. """ @@ -553,4 +555,4 @@ def forward( "eager": H2OLlamaAttention, "flash_attention_2": H2OLlamaFlashAttention2, "sdpa": H2OLlamaSdpaAttention, -} \ No newline at end of file +} diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index a02f0787c94..34cd907909e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -222,8 +222,9 @@ def forward( class H2OOptFlashAttention2(H2OOPTAttention): - """ - OPT flash attention module. This module inherits from `OPTAttention` as the weights of the module stays untouched. + """OPT flash attention module. + + This module inherits from `OPTAttention` as the weights of the module stays untouched. The only required change would be on the forward pass where it needs to correctly call the public API of flash attention and deal with padding tokens in case the input contains any of them. """ @@ -233,7 +234,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() @@ -398,4 +399,4 @@ def forward( if not output_attentions: attn_weights_reshaped = None - return attn_output, attn_weights_reshaped, past_key_value \ No newline at end of file + return attn_output, attn_weights_reshaped, past_key_value From 877329d8bbb997db5cb4fc92848f75e6c14b458f Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Thu, 16 May 2024 21:14:16 -0400 Subject: [PATCH 18/62] change to new api Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/h2o.py | 5 ++- .../models/modeling_bloom.py | 41 ++++++++++++++++--- .../models/modeling_gpt_neox.py | 37 +++++++++++++++-- .../models/modeling_llama.py | 36 ++++++++-------- .../models/modeling_mistral.py | 39 +++++++++++++++--- .../models/modeling_mixtral.py | 39 +++++++++++++++--- .../models/modeling_opt.py | 16 ++++---- 7 files changed, 162 insertions(+), 51 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index a5179a86f24..0864c4dc001 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -190,7 +190,10 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): seq_len = key_states.size(-2) if seq_len <= cache_size: - return key_states, value_states + if self.real_drop: + return key_states, value_states + else: + return torch.ones(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) # hh-selection mask = torch.zeros(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py index 627239e9904..bc5a542c09f 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import math +import logging from torch.nn import functional as F from typing import List, Optional, Tuple, Union @@ -22,7 +23,9 @@ import torch import torch.nn as nn -from ..h2o import get_hh_mask +from ..h2o import H2OKVCache + +logger = logging.getLogger(__name__) class H2OBloomAttention(nn.Module): def __init__( @@ -32,6 +35,9 @@ def __init__( heavy_ratio, recent_ratio, h2o_min_seqlen=1024, + real_drop=False, + is_gen=False, + mean=False ): super().__init__() @@ -59,10 +65,19 @@ def __init__( self.attention_dropout = model.attention_dropout # for h2o + if real_drop: + real_drop = False + logger.error("BloomAttention not support for kv cache, usning simulation mode.") + self.real_drop = real_drop + self.is_gen = is_gen + self.mean = mean + self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory storage as `fused_qkv` @@ -156,13 +171,29 @@ def forward( if input_dtype == torch.float16: attention_scores = attention_scores.to(torch.float) attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.finfo(attention_scores.dtype).min) - # get hh mask - if q_length > self.h2o_min_seqlen: - mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min attention_probs = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(input_dtype) + # get hh mask + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attention_probs, + key_layer, + value_layer, + mean=self.mean + ) + past_key_value = (new_key_states, new_value_states) + else: + mask = self.h2o_kv_cache( + attention_probs, + key_layer, + value_layer, + mean=self.mean + ) + attention_probs = attention_probs * mask.unsqueeze(-2) + # [batch_size, num_heads, q_length, kv_length] attention_probs = self.attention_dropout(attention_probs) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py index f7e9851e814..3c506bebcf4 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -15,12 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. from typing import List, Optional, Tuple, Union +import logging import torch import torch.nn as nn -from ..h2o import get_hh_mask +from ..h2o import H2OKVCache +logger = logging.getLogger(__name__) class GPTNeoXAttention(nn.Module): def __init__( @@ -30,6 +32,9 @@ def __init__( heavy_ratio, recent_ratio, h2o_min_seqlen=1024, + real_drop=False, + is_gen=False, + mean=False ): super().__init__() self.config = config @@ -53,10 +58,19 @@ def __init__( self.is_causal = True # for h2o + if real_drop: + real_drop = False + logger.error("BloomAttention not support for kv cache, usning simulation mode.") + self.real_drop = real_drop + self.is_gen = is_gen + self.mean = mean + self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + def forward( self, hidden_states: torch.FloatTensor, @@ -185,9 +199,24 @@ def _attn(self, query, key, value, attention_mask=None, head_mask=None): attn_weights = attn_weights.to(value.dtype) # get hh mask - if query_length > self.h2o_min_seqlen: - mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - attn_weights[~mask_bottom] = torch.min(attention_mask) + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + key, + value, + mean=self.mean + ) + past_key_value = (new_key_states, new_value_states) + else: + mask = self.h2o_kv_cache( + attn_weights, + key, + value, + mean=self.mean + ) + attn_weights = attn_weights * mask.unsqueeze(1) # Mask heads if we want to if head_mask is not None: diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 2feaf1f8997..18a30d29407 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -48,7 +48,8 @@ def __init__( recent_ratio, h2o_min_seqlen=1024, real_drop=False, - is_gen=False + is_gen=False, + mean=False ): super().__init__() self.config = config @@ -85,6 +86,7 @@ def __init__( # for h2o self.real_drop = real_drop self.is_gen = is_gen + self.mean = mean self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio @@ -159,20 +161,18 @@ def forward( query_states, past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], - attention_mask=attention_mask, - mean=False + mean=self.mean ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states else: mask = self.h2o_kv_cache( attn_weights, - past_key_value[0], - past_key_value[1], - attention_mask=attention_mask, - mean=False + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean ) - attn_weights = attn_weights * mask.unsqueeze(2) + attn_weights = attn_weights * mask.unsqueeze(-2) value_states = value_states * mask.unsqueeze(-1) # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min @@ -303,18 +303,16 @@ def forward( query_states, past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], - attention_mask=attention_mask, - mean=False + mean=self.mean ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states else: mask = self.h2o_kv_cache( attn_weights, - past_key_value[0], - past_key_value[1], - attention_mask=attention_mask, - mean=False + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean ) key_states = key_states * mask.unsqueeze(-1) value_states = value_states * mask.unsqueeze(-1) @@ -515,18 +513,16 @@ def forward( query_states, past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], - attention_mask=attention_mask, - mean=False + mean=self.mean ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states else: mask = self.h2o_kv_cache( attn_weights, - past_key_value[0], - past_key_value[1], - attention_mask=attention_mask, - mean=False + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean ) key_states = key_states * mask.unsqueeze(-1) value_states = value_states * mask.unsqueeze(-1) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index a98a18e1ef0..5972fd739ba 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -24,7 +24,7 @@ from transformers.cache_utils import Cache from transformers.models.mistral.modeling_mistral import apply_rotary_pos_emb, repeat_kv -from ..h2o import get_hh_mask +from ..h2o import H2OKVCache logger = logging.getLogger(__name__) @@ -42,6 +42,9 @@ def __init__( heavy_ratio, recent_ratio, h2o_min_seqlen=1024, + real_drop=False, + is_gen=False, + mean=False ): super().__init__() self.config = config @@ -75,10 +78,16 @@ def __init__( self.rotary_emb = model.rotary_emb # for h2o + self.is_gen = is_gen + self.real_drop = real_drop + self.mean = mean + self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -142,13 +151,31 @@ def forward( attn_weights = attn_weights + attention_mask - # get hh mask - if q_len > self.h2o_min_seqlen: - mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - attn_weights[~mask_bottom] = torch.min(attention_mask) - # upcast attention to fp32 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + + # H2O + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop and past_key_value is not None: + new_key_states, new_value_states = self.h2o_kv_cache( + query_states, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + attn_weights = attn_weights * mask.unsqueeze(-2) + value_states = value_states * mask.unsqueeze(-1) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index 35c04379b20..6d31775a9e7 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -24,7 +24,7 @@ from transformers.cache_utils import Cache from transformers.models.mixtral.modeling_mixtral import apply_rotary_pos_emb, repeat_kv -from ..h2o import get_hh_mask +from ..h2o import H2OKVCache logger = logging.getLogger(__name__) @@ -42,6 +42,9 @@ def __init__( heavy_ratio, recent_ratio, h2o_min_seqlen=1024, + real_drop=False, + is_gen=False, + mean=False ): super().__init__() self.config = config @@ -76,10 +79,16 @@ def __init__( self.rotary_emb = model.rotary_emb # for h2o + self.is_gen = is_gen + self.real_drop = real_drop + self.mean = mean + self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -143,13 +152,31 @@ def forward( attn_weights = attn_weights + attention_mask - # get hh mask - if q_len > self.h2o_min_seqlen: - mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min - # upcast attention to fp32 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + + # H2O + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop and past_key_value is not None: + new_key_states, new_value_states = self.h2o_kv_cache( + query_states, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + attn_weights = attn_weights * mask.unsqueeze(-2) + value_states = value_states * mask.unsqueeze(-1) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index a02f0787c94..9b040a520f6 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -36,7 +36,8 @@ def __init__( recent_ratio, h2o_min_seqlen=1024, real_drop=False, - is_gen=False + is_gen=False, + mean=False ): super().__init__() self.config = config @@ -64,6 +65,7 @@ def __init__( # for h2o self.is_gen = is_gen self.real_drop = real_drop + self.mean = mean self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio @@ -181,8 +183,7 @@ def forward( attn_weights, past_key_value[0], past_key_value[1], - attention_mask=attention_mask, - mean=False + mean=self.mean ) past_key_value = (new_key_states, new_value_states) else: @@ -190,8 +191,7 @@ def forward( attn_weights, past_key_value[0], past_key_value[1], - attention_mask=attention_mask, - mean=False + mean=self.mean ) attn_weights = attn_weights * mask.unsqueeze(1) @@ -373,8 +373,7 @@ def forward( attn_weights, past_key_value[0], past_key_value[1], - attention_mask=attention_mask, - mean=False + mean=self.mean ) past_key_value = (new_key_states, new_value_states) else: @@ -382,8 +381,7 @@ def forward( attn_weights, past_key_value[0], past_key_value[1], - attention_mask=attention_mask, - mean=False + mean=self.mean ) key_states = key_states * mask.unsqueeze(-1) value_states = value_states * mask.unsqueeze(-1) From 50685520bdd4cdb70e5d2d585f0ad5d331d34703 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Fri, 17 May 2024 02:12:28 -0400 Subject: [PATCH 19/62] clean code Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/h2o.py | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 95df3e6db69..d4b878738fd 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -57,6 +57,11 @@ def get_module(model, op_name): module = module return module +def clean_cache(model): + for _, module in model.named_modules(): + if "H2O" in module.__class__.__name__: + module.h2o_kv_cache.clean_scores() + def convert_model( model, heavy_ratio, @@ -80,8 +85,9 @@ def convert_model( "intel_extension_for_transformers.transformers.modeling.kv_cache_compression" ), "H2O" + module.__class__.__name__) - module = h2o_cls(module, module.config, heavy_ratio, recent_ratio, h2o_min_seqlen, real_drop, is_gen) + module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen, real_drop, is_gen) set_module(model, layer_name, module) + model.clean_cache = lambda: clean_cache(model) model = model.to(device) return model @@ -167,7 +173,7 @@ def __init__( real_drop=False, min_seqlen=-1 ): - ## bsz, num_heads, seq_len, head_dim | num_heads, seq_len, head_dim + ## bsz, num_heads, seq_len, head_dim self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.hh_score = None @@ -177,65 +183,51 @@ def __init__( def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): - if key_states.shape[-2] <= self.min_seqlen: + seq_len = key_states.size(-2) + heavy_budget = int(self.heavy_ratio * seq_len) + recent_budget = int(self.recent_ratio * seq_len) + cache_size = heavy_budget + recent_budget + if seq_len <= self.min_seqlen or seq_len <= cache_size: if self.real_drop: return key_states, value_states else: - return torch.ones(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) + return torch.ones(self.attn_score.shape, dtype=attn_score.dtype).to(key_states.device) self.idx += 1 + mask_shape = list(attn_score.shape) + mask_shape.pop(-1) + # attn_score shape (bsz, num_heads, seq_len, head_dim) + if len(attn_score.shape) == 3: + attn_score = attn_score.unsqueeze(0) self._update_hh_score(attn_score, mean=mean) - heavy_budget = int(self.heavy_ratio * self.hh_score.shape[-1]) - recent_budget = int(self.recent_ratio * self.hh_score.shape[-1]) - cache_size = heavy_budget + recent_budget - - seq_len = key_states.size(-2) - if seq_len <= cache_size: - if self.real_drop: - return key_states, value_states - else: - return torch.ones(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) # hh-selection mask = torch.zeros(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) - if len(self.hh_score.shape) == 2: - if not recent_budget == 0: - mask[:,-recent_budget:] = 1 - select_hh_scores = self.hh_score[:, :seq_len - recent_budget] - else: - if not recent_budget == 0: - mask[:,:,-recent_budget:] = 1 - select_hh_scores = self.hh_score[:,:,:seq_len - recent_budget] + if not recent_budget == 0: + mask[:,:,-recent_budget:] = 1 + select_hh_scores = self.hh_score[:,:,:seq_len - recent_budget] if not heavy_budget == 0: _, keep_topk = torch.topk(select_hh_scores, heavy_budget, dim=-1, largest=True) mask = mask.scatter(-1, keep_topk, 1) if not self.real_drop: - return mask + return mask.view(mask_shape) mask = mask.bool() - states_shape = list(key_states.shape) - states_shape[-2] = cache_size - k_hh_recent = key_states[mask].view(*states_shape) - states_shape[-1] = -1 - v_hh_recent = value_states[mask].view(*states_shape) - - tmp_shape = list(self.hh_score.shape) - tmp_shape[-1] = cache_size - self.hh_score = self.hh_score[mask].view(*tmp_shape) + k_hh_recent = key_states[mask].view(key_states.shape[0], key_states.shape[1], cache_size, key_states.shape[3]) + v_hh_recent = value_states[mask].view(value_states.shape[0], value_states.shape[1], cache_size, value_states.shape[3]) + + self.hh_score = self.hh_score[mask].view(self.hh_score.shape[0], self.hh_score.shape[1], cache_size) return k_hh_recent, v_hh_recent def _update_hh_score(self, attn_score_cache, mean=False): - # hh_score size (bsz, num_heads, seq_len) or (num_heads, seq_len) + # hh_score size (bsz, num_heads, head_dim) attn_score_cache = attn_score_cache.sum(-2) if self.hh_score is not None: # clean self.hh_score if not generation mode - if attn_score_cache.shape[-1] < self.hh_score: + if attn_score_cache.size(-1) < self.hh_score.size(-1): self.clean_scores() - elif len(attn_score_cache.shape) == 2: - attn_score_cache[:, :self.hh_score.shape[-1]] += self.hh_score / (1 if not mean else self.idx) - else: attn_score_cache[:, :, :self.hh_score.shape[-1]] += self.hh_score / (1 if not mean else self.idx) self.hh_score = attn_score_cache From 5e5f589721bdc25fbbdaa1874bc183444ba61e32 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 06:13:34 +0000 Subject: [PATCH 20/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../transformers/modeling/kv_cache_compression/h2o.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index d4b878738fd..28f65045fa7 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -173,7 +173,7 @@ def __init__( real_drop=False, min_seqlen=-1 ): - ## bsz, num_heads, seq_len, head_dim + ## bsz, num_heads, seq_len, head_dim self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.hh_score = None From d0dce7d22ffd32dff80b42658b6a3840cdfa5d77 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 20 May 2024 02:11:35 -0400 Subject: [PATCH 21/62] fix Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/h2o.py | 14 +++++++++----- .../models/modeling_bloom.py | 6 +++--- .../models/modeling_gpt_neox.py | 4 ++-- .../models/modeling_llama.py | 18 +++++------------- .../models/modeling_mistral.py | 2 +- .../models/modeling_mixtral.py | 2 +- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index d4b878738fd..b30d12131ca 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -14,8 +14,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import math -from collections import OrderedDict import importlib import torch @@ -193,8 +191,7 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): else: return torch.ones(self.attn_score.shape, dtype=attn_score.dtype).to(key_states.device) self.idx += 1 - mask_shape = list(attn_score.shape) - mask_shape.pop(-1) + mask_shape = attn_score.shape[-1] # attn_score shape (bsz, num_heads, seq_len, head_dim) if len(attn_score.shape) == 3: attn_score = attn_score.unsqueeze(0) @@ -214,10 +211,17 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): return mask.view(mask_shape) mask = mask.bool() + self.hh_score = self.hh_score[mask].view(self.hh_score.shape[0], self.hh_score.shape[1], cache_size) + + # if use repeat_kv, need to reshape mask + n_rep = mask.size(1) / key_states.size(1) + if n_rep > 1: + drop_mask = torch.tensor([True if i % n_rep == 0 else False for i in range(0, mask.size(1))]).repeat(mask.size(0), 1).to(mask.device) + mask = mask[drop_mask].view(key_states.shape[:-1]) + k_hh_recent = key_states[mask].view(key_states.shape[0], key_states.shape[1], cache_size, key_states.shape[3]) v_hh_recent = value_states[mask].view(value_states.shape[0], value_states.shape[1], cache_size, value_states.shape[3]) - self.hh_score = self.hh_score[mask].view(self.hh_score.shape[0], self.hh_score.shape[1], cache_size) return k_hh_recent, v_hh_recent def _update_hh_score(self, attn_score_cache, mean=False): diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py index bc5a542c09f..b58fc06c078 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import math -import logging from torch.nn import functional as F from typing import List, Optional, Tuple, Union @@ -24,8 +23,9 @@ import torch.nn as nn from ..h2o import H2OKVCache +from transformers.utils import logging -logger = logging.getLogger(__name__) +logger = logging.get_logger(__name__) class H2OBloomAttention(nn.Module): def __init__( @@ -67,7 +67,7 @@ def __init__( # for h2o if real_drop: real_drop = False - logger.error("BloomAttention not support for kv cache, usning simulation mode.") + logger.warning_once("BloomAttention not support for kv cache, usning simulation mode.") self.real_drop = real_drop self.is_gen = is_gen self.mean = mean diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py index 3c506bebcf4..559b9ba7d09 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -15,10 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from typing import List, Optional, Tuple, Union -import logging import torch import torch.nn as nn +from transformers.utils import logging from ..h2o import H2OKVCache @@ -60,7 +60,7 @@ def __init__( # for h2o if real_drop: real_drop = False - logger.error("BloomAttention not support for kv cache, usning simulation mode.") + logger.warning_once("BloomAttention not support for kv cache, usning simulation mode.") self.real_drop = real_drop self.is_gen = is_gen self.mean = mean diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 71619b778d8..1e9b87e1960 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -18,13 +18,12 @@ import math from typing import List, Optional, Tuple, Union -import logging import torch import torch.nn as nn import torch.nn.functional as F -from transformers.cache_utils import Cache +from transformers.cache_utils import Cache, logging from transformers.models.llama.configuration_llama import LlamaConfig from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv, _get_unpad_data from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available @@ -35,7 +34,7 @@ from ..h2o import get_hh_mask, H2OKVCache -logger = logging.getLogger(__name__) +logger = logging.get_logger(__name__) class H2OLlamaAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper.""" @@ -158,7 +157,7 @@ def forward( self.h2o_kv_cache.clean_scores() if self.real_drop and past_key_value is not None: new_key_states, new_value_states = self.h2o_kv_cache( - query_states, + attn_weights, past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], mean=self.mean @@ -301,7 +300,7 @@ def forward( self.h2o_kv_cache.clean_scores() if self.real_drop and past_key_value is not None: new_key_states, new_value_states = self.h2o_kv_cache( - query_states, + attn_weights, past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], mean=self.mean @@ -506,13 +505,12 @@ def forward( # upcast attention to fp32 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) if not self.is_gen: self.h2o_kv_cache.clean_scores() if self.real_drop and past_key_value is not None: new_key_states, new_value_states = self.h2o_kv_cache( - query_states, + attn_weights, past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], mean=self.mean @@ -546,9 +544,3 @@ def forward( attn_output = self.o_proj(attn_output) return attn_output, None, past_key_value - -ATTENTION_CLASSES = { - "eager": H2OLlamaAttention, - "flash_attention_2": H2OLlamaFlashAttention2, - "sdpa": H2OLlamaSdpaAttention, -} diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index 5972fd739ba..cd9a179a714 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -159,7 +159,7 @@ def forward( self.h2o_kv_cache.clean_scores() if self.real_drop and past_key_value is not None: new_key_states, new_value_states = self.h2o_kv_cache( - query_states, + attn_weights, past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], mean=self.mean diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index 6d31775a9e7..2626e644ad3 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -160,7 +160,7 @@ def forward( self.h2o_kv_cache.clean_scores() if self.real_drop and past_key_value is not None: new_key_states, new_value_states = self.h2o_kv_cache( - query_states, + attn_weights, past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], mean=self.mean From 24c47252d7f5ac41c2fbb747240ced26b2629574 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 20 May 2024 02:19:33 -0400 Subject: [PATCH 22/62] add example Signed-off-by: n1ck-guo --- .../text-generation/h2o/run_generation.py | 2 - .../text-generation/h2o/run_summarization.py | 163 ++++++++++++++++++ 2 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 examples/huggingface/pytorch/text-generation/h2o/run_summarization.py diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py index cf10e792e2e..b9e0cf605ff 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py @@ -1,6 +1,4 @@ import argparse -import sys -sys.path.insert(0, '/home/hengguo/code/intel-extension-for-transformers') import time import json import torch diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_summarization.py b/examples/huggingface/pytorch/text-generation/h2o/run_summarization.py new file mode 100644 index 00000000000..975438fa0a9 --- /dev/null +++ b/examples/huggingface/pytorch/text-generation/h2o/run_summarization.py @@ -0,0 +1,163 @@ +import argparse +import json +import os.path + +import tqdm +import torch + +from rouge import Rouge +import numpy as np + +from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig +from transformers.models.llama.configuration_llama import LlamaConfig + +from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o import convert_model + + + +os.environ['CUDA_LAUNCH_BLOCKING'] = "1" + +MAX_LENGTH = int(10000) # Hardcoded max length to avoid infinite loop + +def set_seed(args): + np.random.seed(args.seed) + torch.manual_seed(args.seed) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + + parser.add_argument("--input_path", type=str, default="") + parser.add_argument("--output_path", type=str, default="") + + parser.add_argument("--model_name", type=str, default="") + parser.add_argument("--cache_dir", type=str, default=None) + + parser.add_argument('--h2o', action='store_true') + parser.add_argument("--heavy_ratio", type=float, default=0.1) + parser.add_argument("--recent_ratio", type=float, default=0.1) + parser.add_argument("--h2o_min_seqlen", type=int, default=0) + parser.add_argument("--device", type=str, default="cpu") + + + parser.add_argument("--sample_num", type=int, default=100) + parser.add_argument("--k", type=int, default=0) + parser.add_argument("--seed", type=int, default=42, help="random seed for initialization") + parser.add_argument("--batch_size", type=int, default=1) + parser.add_argument( + "--fp16", + action="store_true", + help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit", + ) + args = parser.parse_args() + + try: + device_str = int(args.device) + device_str = f'cuda:{args.device}' + except: + device_str = args.device + set_seed(args) + + model_name = args.model_name + input_path = args.input_path + output_path = args.output_path + + config = AutoConfig.from_pretrained(model_name, cache_dir=args.cache_dir) + tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True, cache_dir=args.cache_dir) + model = AutoModelForCausalLM.from_pretrained(model_name, cache_dir=args.cache_dir) + + if args.batch_size>1: + tokenizer.pad_token = tokenizer.eos_token + + if args.h2o: + print('Enabling H2O KV cache') + model = convert_model( + model, + heavy_ratio=args.heavy_ratio, + recent_ratio=args.recent_ratio, + h2o_min_seqlen=args.h2o_min_seqlen, + real_drop=True, + is_gen=True) + model.clean_cache() + + model = model.half().eval().to(device_str) + + requests = [] + with open(input_path, 'r') as f: + for line in f: + if line.strip() != '': + requests.append(json.loads(line)) + + print(len(requests)) + if args.sample_num < len(requests): + print('Sample {} Examples from {} samples'.format(args.sample_num, len(requests))) + requests = requests[:args.sample_num] + + results = [] + rouge = Rouge() + rouge1_score_list = [] + rouge2_score_list = [] + rougel_score_list = [] + + with torch.no_grad(): + for request in tqdm.tqdm(requests): + result = {'request': request, 'result': {}} + prompt = request['article'] + label = request['summary_gt'] + temperature = request['temperature'] + stop = request['stop'] + + input_ids = tokenizer(prompt, add_special_tokens=False, return_tensors='pt').input_ids.to(model.device) + + output_sequences = model.generate( + input_ids=input_ids, + max_length=request['max_tokens'] + len(input_ids[0]), + temperature=temperature, + top_k=args.k, + top_p=request['top_p'], + do_sample=True, + num_return_sequences=request['n'], + return_dict_in_generate=True, output_scores=True, + pad_token_id=tokenizer.eos_token_id + ) + + if args.h2o: + model.clean_cache() + + tokens = tokenizer.convert_ids_to_tokens(output_sequences['sequences'].squeeze(0))[len(input_ids[0]):] + logprobs = [logits.log_softmax(dim=-1).max().item() for logits in output_sequences['scores']] + top_logprobs = [{i: v for i, v in zip(tokens, logprobs)}] + + generate_text = tokenizer.decode(output_sequences['sequences'].squeeze(0)[len(input_ids[0]):]) + generate_text = generate_text[: generate_text.find(stop[0])] + + scores = rouge.get_scores(generate_text, label)[0] + rouge1_score_list.append(scores['rouge-1']['f']) + rouge2_score_list.append(scores['rouge-2']['f']) + rougel_score_list.append(scores['rouge-l']['f']) + + result['result'] = { + "choices": [ + { + "text": generate_text, + "logprobs": { + "tokens": tokens, + "token_logprobs": logprobs, + "top_logprobs": top_logprobs, + "text_offset": [] + }, + "finish_reason": "length" + } + ], + "request_time": { + "batch_time": 0, + "batch_size": 1} + } + + results.append(result) + print('rouge-1: {:.6f}, rouge-2: {:.6f}, rouge-l: {:.6f}'.format(np.mean(rouge1_score_list), np.mean(rouge2_score_list), np.mean(rougel_score_list))) + + with open(output_path, 'w') as f: + for result in results: + f.write(json.dumps(result) + '\n') \ No newline at end of file From 955e132071aeb2777fc893779912976887e66c4a Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 20 May 2024 02:36:52 -0400 Subject: [PATCH 23/62] clean Signed-off-by: n1ck-guo --- .../text-generation/h2o/generate_task_data.py | 74 ------ .../h2o/run_lm_eval_harness.py | 221 ------------------ .../text-generation/h2o/tasks/__init__.py | 1 - .../text-generation/h2o/tasks/eval_harness.py | 108 --------- .../pytorch/text-generation/h2o/tasks/util.py | 63 ----- 5 files changed, 467 deletions(-) delete mode 100644 examples/huggingface/pytorch/text-generation/h2o/generate_task_data.py delete mode 100644 examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py delete mode 100644 examples/huggingface/pytorch/text-generation/h2o/tasks/__init__.py delete mode 100644 examples/huggingface/pytorch/text-generation/h2o/tasks/eval_harness.py delete mode 100644 examples/huggingface/pytorch/text-generation/h2o/tasks/util.py diff --git a/examples/huggingface/pytorch/text-generation/h2o/generate_task_data.py b/examples/huggingface/pytorch/text-generation/h2o/generate_task_data.py deleted file mode 100644 index 2f44fc8d880..00000000000 --- a/examples/huggingface/pytorch/text-generation/h2o/generate_task_data.py +++ /dev/null @@ -1,74 +0,0 @@ -import argparse -import json - -from lm_eval import evaluator, tasks -from tasks import EvalHarnessAdaptor - - -if __name__ == '__main__': - - - parser = argparse.ArgumentParser( - prog = 'ProgramName', - description = 'What the program does', - epilog = 'Text at the bottom of help') - - parser.add_argument('--output-file', type=str, default='input.jsonl') - parser.add_argument('--task-name', type=str, default='hellaswag') - parser.add_argument('--num-fewshot', type=int, default=0) - args = parser.parse_args() - - seq = 1024 - total_batch = 1 - pe = 'fixed' - - with open(args.output_file, 'w') as f: - pass - - class DryRunner: - def eval(self, batch): - - with open(args.output_file, 'a') as f: - for text in batch['text']: - item = { - "best_of": 1, - "echo": True, - "logprobs": 1, - "max_tokens": 0, - "model": "x", - "n": 1, - "prompt": text, - "request_type": "language-model-inference", - "stop": None, - "temperature": 0, - "top_p": 1 - } - f.write(json.dumps(item) + '\n') - - out = { - 'mask_loss': [1.0] * len(batch), - 'each_correct': [True] * len(batch), - } - return out - - t = DryRunner() - adaptor = EvalHarnessAdaptor(t, seq, total_batch, shrink=pe != "fixed") - results = evaluator.evaluate(adaptor, tasks.get_task_dict([args.task_name - #"lambada_openai", - #"piqa", - #"hellaswag", - #"winogrande", - #"mathqa", - #"pubmedqa", - # "boolq", - # "cb", - # "copa", - # "multirc", - # "record", - # "wic", - # "wsc", - ]), False, args.num_fewshot, None) - print('Finished') - - # dumped = json.dumps(results, indent=2) - # print(dumped) \ No newline at end of file diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py b/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py deleted file mode 100644 index 3f703024f61..00000000000 --- a/examples/huggingface/pytorch/text-generation/h2o/run_lm_eval_harness.py +++ /dev/null @@ -1,221 +0,0 @@ -import argparse -import json, tqdm -import torch -import copy -import os -os.environ["TOKENIZERS_PARALLELISM"] = "false" - -import sys -sys.path.insert(0, '/home/hengguo/code/intel-extension-for-transformers') - -import sys -sys.path.insert(0, '/root/hengguo/intel-extension-for-transformers') - -from lm_eval import evaluator, tasks -from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig - -from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o_sim_drop.modify_llama import convert_kvcache_llama_heavy_recent -from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o_sim_drop.modify_opt import convert_kvcache_opt_heavy_recent -from intel_extension_for_transformers.transformers.modeling.kv_cache_compression.h2o_sim_drop.modify_gptneox import convert_kvcache_gpt_neox_heavy_recent - -from tasks import EvalHarnessAdaptor - -ENABLE_Heavy_Hitter_FUNCTIONS = { - "llama": convert_kvcache_llama_heavy_recent, - "opt": convert_kvcache_opt_heavy_recent, - "gpt_neox": convert_kvcache_gpt_neox_heavy_recent, -} - -if __name__ == '__main__': - - parser = argparse.ArgumentParser( - prog = 'ProgramName', - description = 'What the program does', - epilog = 'Text at the bottom of help') - parser.add_argument("--tasks", nargs='+', default=["lambada_openai", - "hellaswag", "winogrande", "piqa", "wikitext"], - type=str, help="tasks list for accuracy validation") - parser.add_argument('--num_fewshot', type=int, default=0) - - parser.add_argument('--enable_small_cache', action='store_true') - parser.add_argument('--model_name', type=str, default='facebook/opt-350m') - parser.add_argument("--cache_dir", type=str, default=None) - - parser.add_argument("--heavy_ratio", type=float, default=0.1) - parser.add_argument("--recent_ratio", type=float, default=0.1) - parser.add_argument("--device", type=str, default='cpu') - parser.add_argument("--seq_len", type=int, default=1024) - - parser.add_argument('--debug', action='store_true') - args = parser.parse_args() - - batch_size = 1 - pe = 'fixed' - seq = args.seq_len - - # build data - requests = [] - class DryRunner: - def eval(self, batch): - for text in batch['text']: - item = { - "best_of": 1, - "echo": True, - "logprobs": 1, - "max_tokens": 0, - "model": "x", - "n": 1, - "prompt": text, - "request_type": "language-model-inference", - "stop": None, - "temperature": 0, - "top_p": 1 - } - requests.append(item) - out = { - 'mask_loss': [1.0] * len(batch), - 'each_correct': [True] * len(batch), - } - return out - t = DryRunner() - adaptor = EvalHarnessAdaptor(t, seq, batch_size, shrink=pe != "fixed") - result = evaluator.evaluate(adaptor, tasks.get_task_dict(args.tasks), False, args.num_fewshot, None) - - model_name = args.model_name - if 'cpu' in args.device: - device = args.device - else: - device = f"cuda:{args.device}" - - config = AutoConfig.from_pretrained(model_name, cache_dir=args.cache_dir) - tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=args.cache_dir) - model = AutoModelForCausalLM.from_pretrained(model_name, cache_dir=args.cache_dir) - - if args.enable_small_cache: - print('Enable Small Cache Size') - # checkpoint = copy.deepcopy(model.state_dict()) - # model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) - from intel_extension_for_transformers.transformers.modeling.kv_cache_compression import convert_model - model = convert_model(model, heavy_ratio=args.heavy_ratio, recent_ratio=args.recent_ratio, h2o_min_seqlen=0) - # model.load_state_dict(checkpoint) - - model = model.to(device) - print('using device: ', device) - model.eval() - # model.half().eval() - - results = [] - with torch.no_grad(): - for request in tqdm.tqdm(requests): - result = {'request': request, 'result': {}} - prompt = request['prompt'] - input_ids = tokenizer(prompt, add_special_tokens=False, return_tensors='pt').input_ids.to(model.device) - - logits = model(input_ids).logits.log_softmax(dim=-1) - - values, indices = logits.squeeze(0).topk(dim=-1, k=1) - tokens = tokenizer.convert_ids_to_tokens(input_ids.squeeze(0)) - - gold_indices = input_ids[:, 1:] # skip first - logprobs = [None] + torch.gather(logits, -1, gold_indices.unsqueeze(-1)).squeeze(-1).squeeze(0).detach().cpu().tolist() - top_logprobs = [None] + [{tokenizer.convert_ids_to_tokens(i.item()): v.item()} for v, i in zip(values.squeeze(-1), indices.squeeze(-1))] - - result['result'] = { - "choices": [ - { - "text": prompt, - "logprobs": { - "tokens": tokens, - "token_logprobs": logprobs, - "top_logprobs": top_logprobs, - "text_offset": [] - }, - "finish_reason": "length" - } - ], - "request_time": { - "batch_time": 0, - "batch_size": 1} - } - - results.append(result) - - # evaluate - class RealRunner: - def __init__(self, args): - self.results = {} - for item in results: - request = item['request'] - result = item['result'] - self.results[json.dumps(request)] = result - print(f"{len(self.results)} items in the cache") - - def eval(self, batch): - from tasks.eval_harness import tokenizer - mask_loss = [] - each_correct = [] - for i, text in enumerate(batch['text']): - request = { - "best_of": 1, - "echo": True, - "logprobs": 1, - "max_tokens": 0, - "model": "x", - "n": 1, - "prompt": text, - "request_type": "language-model-inference", - "stop": None, - "temperature": 0, - "top_p": 1 - } - - key = json.dumps(request) - correct = True - - if key in self.results: - result = self.results[key] - token_logprobs = result['choices'][0]['logprobs']['token_logprobs'] - tokens = result['choices'][0]['logprobs']['tokens'] - top_logprobs = result['choices'][0]['logprobs']['top_logprobs'] - assert token_logprobs[0] is None - token_ids = tokenizer.convert_tokens_to_ids(tokens) - obs = batch['obs'][i] - target = batch['target'][i] - eval_mask = batch['eval_mask'][i] - - n_positive = 0 - sum_lobprob = 0 - if args.debug: - print(target) - for i, mask in enumerate(eval_mask): - try: - if i+1 >= len(tokens): - break - if mask == True: - if args.debug: - print(tokens[i+1], next(iter(top_logprobs[i+1].keys()))) - correct = correct and (tokens[i+1] == next(iter(top_logprobs[i+1].keys()))) - sum_lobprob += token_logprobs[i+1] - n_positive += 1 - except Exception as e: - raise e - # avg_logprob = sum(token_logprobs[1:]) / (len(token_logprobs) - 1) - avg_logprob = sum_lobprob / n_positive - mask_loss.append( - avg_logprob) - each_correct.append( correct ) - else: - assert False - - out = { - 'mask_loss': mask_loss, - 'each_correct': each_correct, - } - return out - - t = RealRunner(args) - - adaptor = EvalHarnessAdaptor(t, seq, batch_size, shrink=pe != "fixed") - results = evaluator.evaluate(adaptor, tasks.get_task_dict(args.tasks), False, args.num_fewshot, None) - - dumped = json.dumps(results, indent=2) - print(dumped) diff --git a/examples/huggingface/pytorch/text-generation/h2o/tasks/__init__.py b/examples/huggingface/pytorch/text-generation/h2o/tasks/__init__.py deleted file mode 100644 index f8ef348d72a..00000000000 --- a/examples/huggingface/pytorch/text-generation/h2o/tasks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from tasks.eval_harness import EvalHarnessAdaptor \ No newline at end of file diff --git a/examples/huggingface/pytorch/text-generation/h2o/tasks/eval_harness.py b/examples/huggingface/pytorch/text-generation/h2o/tasks/eval_harness.py deleted file mode 100644 index 94528f13bec..00000000000 --- a/examples/huggingface/pytorch/text-generation/h2o/tasks/eval_harness.py +++ /dev/null @@ -1,108 +0,0 @@ -from functools import partial - -import os -import transformers -from lm_eval.base import LM -from tqdm import tqdm -import numpy as np - -from .util import sample_batch, shrink_seq -import multiprocessing -import ftfy - -from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig - -tokenizer = None - -def process_init(): - global tokenizer - model_name = os.environ.get('MODEL_NAME', 'facebook/opt-1.3b') - - if model_name == "EleutherAI/gpt-neox-20b": - tokenizer = AutoTokenizer.from_pretrained(model_name) - tokenizer.model_max_length = int(1e30) - tokenizer.pad_token = "<|endoftext|>" - elif model_name == 'huggyllama/llama-7b': - tokenizer = AutoTokenizer.from_pretrained(model_name) - tokenizer.model_max_length = int(1e30) - tokenizer.pad_token = "<|endoftext|>" - else: - tokenizer = AutoTokenizer.from_pretrained(model_name) - tokenizer.add_bos_token = False - -def process_request(x, seq): - global tokenizer - - ctx, cont = x -# ctx_tokens = tokenizer.encode("<|endoftext|>" + ftfy.fix_text(ctx, normalization="NFKC")) - ctx_text = ftfy.fix_text(ctx, normalization="NFKC") - cont_text = ftfy.fix_text(cont, normalization="NFKC") - all_text = ctx_text + cont_text - - ctx_tokens = tokenizer(ctx_text, add_special_tokens=False)['input_ids'] - cont_tokens = tokenizer(cont_text, add_special_tokens=False)['input_ids'] - - all_tokens = ctx_tokens + cont_tokens - all_tokens = np.array(all_tokens)[-seq:] # truncate sequence at seq length - - provided_ctx = len(all_tokens) - 1 - pad_amount = seq - provided_ctx - - return { - "obs": np.pad(all_tokens[:-1], ((0, pad_amount),), constant_values=tokenizer.pad_token_id), - "target": np.pad(all_tokens[1:], ((0, pad_amount),), constant_values=tokenizer.pad_token_id), - "ctx_length": seq, - "eval_mask": np.logical_and( - np.arange(0, seq) > len(all_tokens) - len(cont_tokens) - 2, - np.arange(0, seq) < len(all_tokens) - 1 - ), - "prompt": ctx_text, - "target": cont_text, - "text": all_text, - } - - -class EvalHarnessAdaptor(LM): - def greedy_until(self, requests): - raise Exception("unimplemented") - - def loglikelihood_rolling(self, requests): - raise Exception("unimplemented") - - def __init__(self, tpu_cluster, seq, batch, shrink, min_seq=None): - super().__init__() - self.tpu = tpu_cluster - self.seq = seq - self.batch = batch - self.shrink = shrink - self.min_seq = min_seq - - self.pool = multiprocessing.Pool(processes=1, initializer=process_init) - # self.pool = multiprocessing.Pool(initializer=process_init) - process_init() - - def convert_requests(self, requests): - return self.pool.imap(partial(process_request, seq=self.seq), requests) - - def loglikelihood(self, requests): - output = [] - - r = self.convert_requests(requests) - self.pool.close() - zero_example = process_request(requests[0], self.seq) - - for b in tqdm(sample_batch(r, self.batch, zero_example), - desc="LM eval harness", - total=len(requests) // self.batch): - - if self.shrink: - b = shrink_seq(b, min_seq=self.min_seq) - - out = self.tpu.eval(b) - - for loss, correct in zip(out["mask_loss"], out["each_correct"]): - output.append((float(-loss), bool(correct))) - - return output - - diff --git a/examples/huggingface/pytorch/text-generation/h2o/tasks/util.py b/examples/huggingface/pytorch/text-generation/h2o/tasks/util.py deleted file mode 100644 index 6cf9080d3d8..00000000000 --- a/examples/huggingface/pytorch/text-generation/h2o/tasks/util.py +++ /dev/null @@ -1,63 +0,0 @@ -from itertools import zip_longest - -import numpy as np - - -def grouper(n, iterable, fillvalue): - "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" - args = [iter(iterable)] * n - return zip_longest(fillvalue=fillvalue, *args) - - -# divide the seq length by 2 until it would truncate actual context -def shrink_seq(examples, min_seq=None): - length = examples["obs"].shape[-1] - - new_length = length // 2 - - if min_seq is not None: - if new_length < min_seq: - return examples - - max_length = np.max(examples["eval_mask"] * np.arange(0, length)) + 1 - - if max_length < new_length: - examples["obs"] = examples["obs"][:, :new_length] - examples["target"] = examples["target"][:, :new_length] - examples["eval_mask"] = examples["eval_mask"][:, :new_length] - - return shrink_seq(examples, min_seq=min_seq) - else: - return examples - - -def sample_batch(examples, bs, zero_example_shape): - zero_example = { - "obs": np.zeros_like(zero_example_shape["obs"]), - "target": np.zeros_like(zero_example_shape["target"]), - "eval_mask": np.zeros_like(zero_example_shape["eval_mask"]), - "ctx_length": 0, - } - - for batch in grouper(bs, examples, zero_example): - batch_flattened = { - "obs": [], - "target": [], - "eval_mask": [], - "ctx_length": [], - "text": [], - } - - for sample in batch: - batch_flattened["obs"].append(sample["obs"]) - batch_flattened["target"].append(sample["target"]) - batch_flattened["eval_mask"].append(sample["eval_mask"]) - batch_flattened["ctx_length"].append(sample["ctx_length"]) - batch_flattened["text"].append(sample["text"]) - - batch_flattened["obs"] = np.array(batch_flattened["obs"]) - batch_flattened["target"] = np.array(batch_flattened["target"]) - batch_flattened["eval_mask"] = np.array(batch_flattened["eval_mask"]) - batch_flattened["ctx_length"] = np.array(batch_flattened["ctx_length"]) - - yield batch_flattened \ No newline at end of file From 91efe57d5fb8983eff0089fbc064bf69cc4349d7 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 20 May 2024 03:48:21 -0400 Subject: [PATCH 24/62] pylint Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/h2o.py | 52 ++++++++++------ .../models/modeling_bloom.py | 1 - .../models/modeling_gpt_neox.py | 2 +- .../models/modeling_llama.py | 61 +++++++++++++------ .../models/modeling_mistral.py | 3 +- .../models/modeling_mixtral.py | 3 +- .../models/modeling_opt.py | 6 +- 7 files changed, 84 insertions(+), 44 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 94ef3ea3e7a..568201bb9a5 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -66,7 +66,8 @@ def convert_model( recent_ratio, h2o_min_seqlen=1024, real_drop=True, - is_gen=False + is_gen=False, + mean=False ): model_type = model.config.model_type device = model.device @@ -83,7 +84,16 @@ def convert_model( "intel_extension_for_transformers.transformers.modeling.kv_cache_compression" ), "H2O" + module.__class__.__name__) - module = h2o_cls(module, model.config, heavy_ratio, recent_ratio, h2o_min_seqlen, real_drop, is_gen) + module = h2o_cls( + module, + model.config, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=h2o_min_seqlen, + real_drop=real_drop, + is_gen=is_gen, + mean=mean + ) set_module(model, layer_name, module) model.clean_cache = lambda: clean_cache(model) model = model.to(device) @@ -94,7 +104,8 @@ def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=No # attn_weights (head, query, keys) or (BS, head, query, keys) attn_shape_len = len(attn_weights.shape) - assert attn_shape_len in [3,4], "Wrong shape of attn_weights. Should be (head, query, keys) or (BS, head, query, keys)" + assert attn_shape_len in [3,4], \ + "Wrong shape of attn_weights. Should be (head, query, keys) or (BS, head, query, keys)" dtype_attn_weights = attn_weights.dtype seq_length = attn_weights.shape[-1] if no_padding_seq_length is None: @@ -106,25 +117,31 @@ def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=No tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) if len(attn_weights.shape) == 3: - accumulated_attention_score = torch.sum(tmp_attn[:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) + accumulated_attention_score = torch.sum( + tmp_attn[:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) accumulated_attention_score[:,heavy_budget+padding_length:] = 0 if padding_length > 0: accumulated_attention_score[:,:padding_length] = 0 mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - mask_bottom[:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True + mask_bottom[:,padding_length:heavy_budget+padding_length, + padding_length:heavy_budget+padding_length] = True else: - accumulated_attention_score = torch.sum(tmp_attn[:,:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) + accumulated_attention_score = torch.sum( + tmp_attn[:,:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) accumulated_attention_score[:,:,heavy_budget+padding_length:] = 0 accumulated_attention_score[:,:,:padding_length] = 0 mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - mask_bottom[:,:, padding_length:heavy_budget+padding_length, padding_length:heavy_budget+padding_length] = True + mask_bottom[:,:, padding_length:heavy_budget+padding_length, + padding_length:heavy_budget+padding_length] = True for token_index in range(heavy_budget+padding_length, seq_length): if attn_shape_len == 3: - tmp_attn_index = nn.functional.softmax(attn_weights[:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + tmp_attn_index = nn.functional.softmax( + attn_weights[:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) else: - tmp_attn_index = nn.functional.softmax(attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + tmp_attn_index = nn.functional.softmax( + attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) @@ -146,7 +163,8 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): heavy_budget = int(heavy_budget_ratio * attn_weights.shape[-1]) recent_budget = int(recent_budget_ratio * attn_weights.shape[-1]) if heavy_budget > 0: - mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) # Default: No padding applied to input + # Default: No padding applied to input + mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) else: mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) @@ -157,10 +175,6 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): # Combine h2o+recent and apply casual mask mask_bottom = torch.tril(mask_bottom, diagonal=0) - # ones = torch.ones_like(attn_weights, dtype=torch.bool) - # ones = torch.tril(ones, diagonal=recent_budget) - # ones = torch.triu(ones, diagonal=-recent_budget) - # mask_bottom = torch.logical_or(mask_bottom, ones) return mask_bottom class H2OKVCache: @@ -189,7 +203,7 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): if self.real_drop: return key_states, value_states else: - return torch.ones(self.attn_score.shape, dtype=attn_score.dtype).to(key_states.device) + return torch.ones(attn_score.shape, dtype=attn_score.dtype).to(key_states.device) self.idx += 1 mask_shape = attn_score.shape[-1] # attn_score shape (bsz, num_heads, seq_len, head_dim) @@ -216,11 +230,13 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): # if use repeat_kv, need to reshape mask n_rep = mask.size(1) / key_states.size(1) if n_rep > 1: - drop_mask = torch.tensor([True if i % n_rep == 0 else False for i in range(0, mask.size(1))]).repeat(mask.size(0), 1).to(mask.device) + drop_mask = torch.tensor( + [True if i % n_rep == 0 else False for i in range(0, mask.size(1))] + ).repeat(mask.size(0), 1).to(mask.device) mask = mask[drop_mask].view(key_states.shape[:-1]) - k_hh_recent = key_states[mask].view(key_states.shape[0], key_states.shape[1], cache_size, key_states.shape[3]) - v_hh_recent = value_states[mask].view(value_states.shape[0], value_states.shape[1], cache_size, value_states.shape[3]) + k_hh_recent = key_states[mask].view(key_states.shape[0], key_states.shape[1], cache_size, -1) + v_hh_recent = value_states[mask].view(value_states.shape[0], value_states.shape[1], cache_size, -1) return k_hh_recent, v_hh_recent diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py index b58fc06c078..ec82b5de563 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -165,7 +165,6 @@ def forward( # change view to [batch_size, num_heads, q_length, kv_length] attention_scores = matmul_result.view(batch_size, self.num_heads, q_length, kv_length) - # cast attention scores to fp32, compute scaled softmax and cast back to initial dtype - [batch_size, num_heads, q_length, kv_length] input_dtype = attention_scores.dtype # `float16` has a minimum value of -65504.0, whereas `bfloat16` and `float32` have a minimum value of `-3.4e+38` if input_dtype == torch.float16: diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py index 559b9ba7d09..bc0110f54f8 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -22,7 +22,7 @@ from ..h2o import H2OKVCache -logger = logging.getLogger(__name__) +logger = logging.get_logger(__name__) class GPTNeoXAttention(nn.Module): def __init__( diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 1e9b87e1960..8e7fd920aaa 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -23,18 +23,22 @@ import torch.nn as nn import torch.nn.functional as F -from transformers.cache_utils import Cache, logging +from transformers.cache_utils import Cache from transformers.models.llama.configuration_llama import LlamaConfig from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv, _get_unpad_data -from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available +from transformers.utils import logging, is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available + +logger = logging.get_logger(__name__) if is_flash_attn_2_available(): - from flash_attn import flash_attn_func, flash_attn_varlen_func - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + try: + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + except: + logging.error("Unable to import flash_attn. Make sure it is installed.") -from ..h2o import get_hh_mask, H2OKVCache +from ..h2o import H2OKVCache -logger = logging.get_logger(__name__) class H2OLlamaAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper.""" @@ -132,7 +136,17 @@ def forward( value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) past_key_value = getattr(self, "past_key_value", past_key_value) - cos, sin = self.rotary_emb(value_states, position_ids) + try: + cos, sin = self.rotary_emb(value_states, position_ids) + except: # for old version + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + position_length = kv_seq_len + if not position_ids.nelement() > 1: + if position_length < position_ids.item()+1: + position_length = position_ids.item()+1 + cos, sin = self.rotary_emb(value_states, seq_len=position_length) query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_value is not None: @@ -212,8 +226,11 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, + # that was made default for flash_attn>=2.1. This attribute is used to handle this difference. + # Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, + # using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() def forward( @@ -252,8 +269,6 @@ def forward( cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache - # to be able to avoid many of these transpose/reshape/view. query_states = query_states.transpose(1, 2) key_states = key_states.transpose(1, 2) value_states = value_states.transpose(1, 2) @@ -354,7 +369,6 @@ def _flash_attention_forward( if not self._flash_attn_uses_top_left_mask: causal = self.is_causal else: - # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. causal = self.is_causal and query_length != 1 # Contains at least one padding token in the sequence @@ -448,8 +462,10 @@ def forward( if output_attentions: # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. logger.warning_once( - "LlamaModel is using LlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " - 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + "LlamaModel is using LlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` " + "does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards.' + '"This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' ) return super().forward( hidden_states=hidden_states, @@ -471,7 +487,17 @@ def forward( key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - cos, sin = self.rotary_emb(value_states, position_ids) + try: + cos, sin = self.rotary_emb(value_states, position_ids) + except: + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + position_length = kv_seq_len + if not position_ids.nelement() > 1: + if position_length < position_ids.item()+1: + position_length = position_ids.item()+1 + cos, sin = self.rotary_emb(value_states, seq_len=position_length) query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) # In case static cache is used, it is an instance attribute. @@ -489,8 +515,6 @@ def forward( if attention_mask is not None: causal_mask = causal_mask[:, :, :, : key_states.shape[-2]] - # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, - # Reference: https://github.com/pytorch/pytorch/issues/112577. if query_states.device.type == "cuda" and causal_mask is not None: query_states = query_states.contiguous() key_states = key_states.contiguous() @@ -527,7 +551,8 @@ def forward( key_states = key_states * mask.unsqueeze(-1) value_states = value_states * mask.unsqueeze(-1) - # In case we are not compiling, we may set `causal_mask` to None, which is required to dispatch to SDPA's Flash Attention 2 backend, rather + # In case we are not compiling, we may set `causal_mask` to None, + # which is required to dispatch to SDPA's Flash Attention 2 backend, rather # relying on the `is_causal` argument. attn_output = torch.nn.functional.scaled_dot_product_attention( query_states, diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index cd9a179a714..e912984cb11 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -103,7 +103,8 @@ def forward( ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: if "padding_mask" in kwargs: logger.warn( - "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + "Passing `padding_mask` is deprecated and will be removed in v4.37. " + "Please make sure use `attention_mask` instead.`" ) bsz, q_len, _ = hidden_states.size() diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index 2626e644ad3..0295e27c32b 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -104,7 +104,8 @@ def forward( ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: if "padding_mask" in kwargs: logger.warn( - "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + "Passing `padding_mask` is deprecated and will be removed in v4.37." + "Please make sure use `attention_mask` instead.`" ) bsz, q_len, _ = hidden_states.size() diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index 9ae96e4dfa8..e95c2fed7bf 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -212,7 +212,8 @@ def forward( attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) attn_output = attn_output.transpose(1, 2) - # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # Use the `embed_dim` from the config (stored in the class) + # rather than `hidden_state` because `attn_output` can be # partitioned aross GPUs when using tensor-parallelism. attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) @@ -233,9 +234,6 @@ class H2OOptFlashAttention2(H2OOPTAttention): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() def forward( From 4190edbf6c1ae9894561a8d4ed273f02c7f436cc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 07:49:36 +0000 Subject: [PATCH 25/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../modeling/kv_cache_compression/models/modeling_mixtral.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index 0295e27c32b..cc0a82019f7 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -104,7 +104,7 @@ def forward( ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: if "padding_mask" in kwargs: logger.warn( - "Passing `padding_mask` is deprecated and will be removed in v4.37." + "Passing `padding_mask` is deprecated and will be removed in v4.37." "Please make sure use `attention_mask` instead.`" ) bsz, q_len, _ = hidden_states.size() From e71bf92432aba289c49cfb22d9d4a466f1470ce0 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 20 May 2024 04:23:08 -0400 Subject: [PATCH 26/62] pylint Signed-off-by: n1ck-guo --- .../kv_cache_compression/models/modeling_llama.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 8e7fd920aaa..865ff82c550 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -136,9 +136,9 @@ def forward( value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) past_key_value = getattr(self, "past_key_value", past_key_value) - try: + try: # pylint: disable=E1120 cos, sin = self.rotary_emb(value_states, position_ids) - except: # for old version + except: # for old version kv_seq_len = key_states.shape[-2] if past_key_value is not None: kv_seq_len += past_key_value[0].shape[-2] @@ -146,7 +146,7 @@ def forward( if not position_ids.nelement() > 1: if position_length < position_ids.item()+1: position_length = position_ids.item()+1 - cos, sin = self.rotary_emb(value_states, seq_len=position_length) + cos, sin = self.rotary_emb(value_states, seq_len=position_length) # pylint: disable=E1120 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_value is not None: @@ -460,7 +460,6 @@ def forward( cache_position: Optional[torch.LongTensor] = None, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: if output_attentions: - # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. logger.warning_once( "LlamaModel is using LlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` " "does not support `output_attentions=True`. Falling back to the manual attention implementation, " @@ -488,7 +487,7 @@ def forward( value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) try: - cos, sin = self.rotary_emb(value_states, position_ids) + cos, sin = self.rotary_emb(value_states, position_ids) # pylint: disable=E1120 except: kv_seq_len = key_states.shape[-2] if past_key_value is not None: @@ -497,7 +496,7 @@ def forward( if not position_ids.nelement() > 1: if position_length < position_ids.item()+1: position_length = position_ids.item()+1 - cos, sin = self.rotary_emb(value_states, seq_len=position_length) + cos, sin = self.rotary_emb(value_states, seq_len=position_length) # pylint: disable=E1120 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) # In case static cache is used, it is an instance attribute. From 41f016cf8f226d0963b4f36bd520c8788a3f1c8f Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 20 May 2024 04:24:13 -0400 Subject: [PATCH 27/62] pylint Signed-off-by: n1ck-guo --- .../kv_cache_compression/models/modeling_llama.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 865ff82c550..cb6f9a91983 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -30,12 +30,9 @@ logger = logging.get_logger(__name__) -if is_flash_attn_2_available(): - try: - from flash_attn import flash_attn_func, flash_attn_varlen_func - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - except: - logging.error("Unable to import flash_attn. Make sure it is installed.") +if is_flash_attn_2_available(): # pylint: disable=E0401 + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa from ..h2o import H2OKVCache From d49487fdfd77d44a9e15015abd3a94a200d6fb83 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 08:25:29 +0000 Subject: [PATCH 28/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../modeling/kv_cache_compression/models/modeling_llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index cb6f9a91983..0cf941d25e0 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -135,7 +135,7 @@ def forward( past_key_value = getattr(self, "past_key_value", past_key_value) try: # pylint: disable=E1120 cos, sin = self.rotary_emb(value_states, position_ids) - except: # for old version + except: # for old version kv_seq_len = key_states.shape[-2] if past_key_value is not None: kv_seq_len += past_key_value[0].shape[-2] From 9ac5ecaf9cfae9fd2d0b0c0246cbead0a9a162e7 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 20 May 2024 22:58:18 -0400 Subject: [PATCH 29/62] fix import error Signed-off-by: n1ck-guo --- .../models/modeling_llama.py | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index cb6f9a91983..8de71a43815 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -16,6 +16,7 @@ # limitations under the License. """PyTorch llama model.""" +import os import math from typing import List, Optional, Tuple, Union @@ -25,17 +26,45 @@ from transformers.cache_utils import Cache from transformers.models.llama.configuration_llama import LlamaConfig -from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv, _get_unpad_data -from transformers.utils import logging, is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available +from transformers.models.llama.modeling_llama import rotate_half, repeat_kv, _get_unpad_data logger = logging.get_logger(__name__) -if is_flash_attn_2_available(): # pylint: disable=E0401 - from flash_attn import flash_attn_func, flash_attn_varlen_func - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa +from packaging import version +import transformers +if version.parse(transformers.__version__) > version.parse("4.33.0"): + from transformers.utils import logging, is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available + if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa from ..h2o import H2OKVCache +def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`, *optional*): + Deprecated and unused. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed class H2OLlamaAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper.""" @@ -144,7 +173,7 @@ def forward( if position_length < position_ids.item()+1: position_length = position_ids.item()+1 cos, sin = self.rotary_emb(value_states, seq_len=position_length) # pylint: disable=E1120 - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) if past_key_value is not None: # sin and cos are specific to RoPE models; cache_position needed for the static cache From 09def0b1d1dd0423458d17813b16a5a9e79464d3 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 20 May 2024 23:03:27 -0400 Subject: [PATCH 30/62] update Signed-off-by: n1ck-guo --- .../transformers/modeling/kv_cache_compression/h2o.py | 7 ++++++- .../modeling/kv_cache_compression/models/modeling_llama.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 568201bb9a5..6707c64f8cc 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -248,10 +248,15 @@ def _update_hh_score(self, attn_score_cache, mean=False): # clean self.hh_score if not generation mode if attn_score_cache.size(-1) < self.hh_score.size(-1): self.clean_scores() - attn_score_cache[:, :, :self.hh_score.shape[-1]] += self.hh_score / (1 if not mean else self.idx) + if not mean: + attn_score_cache[:, :, :self.hh_score.shape[-1]] += self.hh_score + else: + attn_score_cache[:, :, :self.hh_score.shape[-1]] = attn_score_cache[:, :, :self.hh_score.shape[-1]] * (self.idx - 1) + self.hh_score + attn_score_cache /= self.idx self.hh_score = attn_score_cache def clean_scores(self): + self.idx = 0 self.hh_score = None diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 8de71a43815..edcd10b1184 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -24,7 +24,7 @@ import torch.nn as nn import torch.nn.functional as F -from transformers.cache_utils import Cache +from transformers.cache_utils import Cache, logging from transformers.models.llama.configuration_llama import LlamaConfig from transformers.models.llama.modeling_llama import rotate_half, repeat_kv, _get_unpad_data From 3042dd4f2c0dfc32dcaaf7d9c7bc410ef1411a10 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 03:04:36 +0000 Subject: [PATCH 31/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../transformers/modeling/kv_cache_compression/h2o.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 6707c64f8cc..248635d75f7 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -249,9 +249,9 @@ def _update_hh_score(self, attn_score_cache, mean=False): if attn_score_cache.size(-1) < self.hh_score.size(-1): self.clean_scores() if not mean: - attn_score_cache[:, :, :self.hh_score.shape[-1]] += self.hh_score + attn_score_cache[:, :, :self.hh_score.shape[-1]] += self.hh_score else: - attn_score_cache[:, :, :self.hh_score.shape[-1]] = attn_score_cache[:, :, :self.hh_score.shape[-1]] * (self.idx - 1) + self.hh_score + attn_score_cache[:, :, :self.hh_score.shape[-1]] = attn_score_cache[:, :, :self.hh_score.shape[-1]] * (self.idx - 1) + self.hh_score attn_score_cache /= self.idx self.hh_score = attn_score_cache From 3a992abc9fd3cb04b99ac8cf388ef248e02e68da Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 21 May 2024 01:55:23 -0400 Subject: [PATCH 32/62] pylint Signed-off-by: n1ck-guo --- .../text-generation/h2o/run_generation.py | 3 +++ .../text-generation/h2o/run_summarization.py | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py index b9e0cf605ff..b41712d9d96 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py @@ -1,4 +1,6 @@ import argparse +import sys +sys.path.insert(0, '/home/hengguo/code/intel-extension-for-transformers') import time import json import torch @@ -195,6 +197,7 @@ from intel_extension_for_transformers.transformers.llm.evaluation.lm_eval import evaluate, LMEvalParser model_args="pretrained="+args.model+",trust_remote_code="+str(args.trust_remote_code) args.tasks = ",".join(args.tasks) + tokenizer.pad_token = tokenizer.eos_token eval_args = LMEvalParser(model = "hf", user_model=user_model, tokenizer=tokenizer, diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_summarization.py b/examples/huggingface/pytorch/text-generation/h2o/run_summarization.py index 975438fa0a9..ba89c5a73a6 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_summarization.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_summarization.py @@ -1,6 +1,8 @@ import argparse import json import os.path +import sys +sys.path.insert(0, '/home/hengguo/code/intel-extension-for-transformers') import tqdm import torch @@ -23,8 +25,7 @@ def set_seed(args): np.random.seed(args.seed) torch.manual_seed(args.seed) - -if __name__ == '__main__': +def main(): parser = argparse.ArgumentParser() @@ -39,6 +40,7 @@ def set_seed(args): parser.add_argument("--recent_ratio", type=float, default=0.1) parser.add_argument("--h2o_min_seqlen", type=int, default=0) parser.add_argument("--device", type=str, default="cpu") + parser.add_argument('--mean', action='store_true') parser.add_argument("--sample_num", type=int, default=100) @@ -78,7 +80,8 @@ def set_seed(args): recent_ratio=args.recent_ratio, h2o_min_seqlen=args.h2o_min_seqlen, real_drop=True, - is_gen=True) + is_gen=True, + mean=args.mean) model.clean_cache() model = model.half().eval().to(device_str) @@ -156,8 +159,11 @@ def set_seed(args): } results.append(result) - print('rouge-1: {:.6f}, rouge-2: {:.6f}, rouge-l: {:.6f}'.format(np.mean(rouge1_score_list), np.mean(rouge2_score_list), np.mean(rougel_score_list))) + print('rouge-1: {:.6f}, rouge-2: {:.6f}, rouge-l: {:.6f}, prompt length: {}, generate text length: {}'.format(np.mean(rouge1_score_list), np.mean(rouge2_score_list), np.mean(rougel_score_list), input_ids.size(-1), output_sequences['sequences'].size(-1))) with open(output_path, 'w') as f: for result in results: - f.write(json.dumps(result) + '\n') \ No newline at end of file + f.write(json.dumps(result) + '\n') + +if __name__ == "__main__": + main() \ No newline at end of file From 8c89cbc265b95cc6d054fc0e00ee7b280902fba9 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 21 May 2024 01:55:58 -0400 Subject: [PATCH 33/62] pylint Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/h2o.py | 5 +- .../models/modeling_llama.py | 596 ------------------ 2 files changed, 3 insertions(+), 598 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 6707c64f8cc..263542ef160 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -205,7 +205,7 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): else: return torch.ones(attn_score.shape, dtype=attn_score.dtype).to(key_states.device) self.idx += 1 - mask_shape = attn_score.shape[-1] + mask_shape = attn_score.shape[:-1] # attn_score shape (bsz, num_heads, seq_len, head_dim) if len(attn_score.shape) == 3: attn_score = attn_score.unsqueeze(0) @@ -251,7 +251,8 @@ def _update_hh_score(self, attn_score_cache, mean=False): if not mean: attn_score_cache[:, :, :self.hh_score.shape[-1]] += self.hh_score else: - attn_score_cache[:, :, :self.hh_score.shape[-1]] = attn_score_cache[:, :, :self.hh_score.shape[-1]] * (self.idx - 1) + self.hh_score + attn_score_cache[:,:,:self.hh_score.shape[-1]] = attn_score_cache[:,:,:self.hh_score.shape[-1]] \ + * (self.idx - 1) + self.hh_score attn_score_cache /= self.idx self.hh_score = attn_score_cache diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index da141fc70b8..e69de29bb2d 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -1,596 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""PyTorch llama model.""" - -import os -import math -from typing import List, Optional, Tuple, Union - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from transformers.cache_utils import Cache, logging -from transformers.models.llama.configuration_llama import LlamaConfig -from transformers.models.llama.modeling_llama import rotate_half, repeat_kv, _get_unpad_data - -logger = logging.get_logger(__name__) - -from packaging import version -import transformers -if version.parse(transformers.__version__) > version.parse("4.33.0"): - from transformers.utils import logging, is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available - if is_flash_attn_2_available(): - from flash_attn import flash_attn_func, flash_attn_varlen_func - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - -from ..h2o import H2OKVCache - -def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): - """Applies Rotary Position Embedding to the query and key tensors. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - position_ids (`torch.Tensor`, *optional*): - Deprecated and unused. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - cos = cos.unsqueeze(unsqueeze_dim) - sin = sin.unsqueeze(unsqueeze_dim) - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - -class H2OLlamaAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper.""" - - def __init__( - self, - model, - config: LlamaConfig, - heavy_ratio, - recent_ratio, - h2o_min_seqlen=1024, - real_drop=False, - is_gen=False, - mean=False - ): - super().__init__() - self.config = config - self.layer_idx = model.layer_idx - if self.layer_idx is None: - logger.warning_once( - f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " - "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " - "when creating this class." - ) - - self.attention_dropout = config.attention_dropout - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.max_position_embeddings = config.max_position_embeddings - self.rope_theta = config.rope_theta - self.is_causal = True - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError( - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads})." - ) - - self.q_proj = model.q_proj - self.k_proj = model.k_proj - self.v_proj = model.v_proj - self.o_proj = model.o_proj - self.rotary_emb = model.rotary_emb - - # for h2o - self.real_drop = real_drop - self.is_gen = is_gen - self.mean = mean - - self.heavy_ratio = heavy_ratio - self.recent_ratio = recent_ratio - self.h2o_min_seqlen = h2o_min_seqlen - - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, # transformers.cache_utils.DynamicCache - output_attentions: bool = False, - use_cache: bool = False, - cache_position: Optional[torch.LongTensor] = None, - **kwargs, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - bsz, q_len, _ = hidden_states.size() - if self.config.pretraining_tp > 1: - key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp - query_slices = self.q_proj.weight.split( - (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 - ) - key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) - value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) - - query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.config.pretraining_tp)] - query_states = torch.cat(query_states, dim=-1) - - key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.config.pretraining_tp)] - key_states = torch.cat(key_states, dim=-1) - - value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.config.pretraining_tp)] - value_states = torch.cat(value_states, dim=-1) - - else: - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - past_key_value = getattr(self, "past_key_value", past_key_value) - try: # pylint: disable=E1120 - cos, sin = self.rotary_emb(value_states, position_ids) - except: # for old version - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - kv_seq_len += past_key_value[0].shape[-2] - position_length = kv_seq_len - if not position_ids.nelement() > 1: - if position_length < position_ids.item()+1: - position_length = position_ids.item()+1 - cos, sin = self.rotary_emb(value_states, seq_len=position_length) # pylint: disable=E1120 - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - - if past_key_value is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attention_mask is not None: # no matter the length, we just slice it - causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] - attn_weights = attn_weights + causal_mask - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) - - # H2O - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop and past_key_value is not None: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - attn_weights = attn_weights * mask.unsqueeze(-2) - value_states = value_states * mask.unsqueeze(-1) - # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min - - attn_output = torch.matmul(attn_weights, value_states) - - if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) - - if self.config.pretraining_tp > 1: - attn_output = attn_output.split(self.hidden_size // self.config.pretraining_tp, dim=2) - o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.config.pretraining_tp, dim=1) - attn_output = sum([F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.config.pretraining_tp)]) - else: - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - -class H2OLlamaFlashAttention2(H2OLlamaAttention): - """Llama flash attention module. - - This module inherits from `LlamaAttention` as the weights of the module stays - untouched. The only required change would be on the forward pass where it needs to correctly call the public API of - flash attention and deal with padding tokens in case the input contains any of them. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, - # that was made default for flash_attn>=2.1. This attribute is used to handle this difference. - # Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, - # using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.LongTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - cache_position: Optional[torch.LongTensor] = None, - **kwargs, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - output_attentions = False - - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - # Flash attention requires the input to have the shape - # batch_size x seq_length x head_dim x hidden_dim - # therefore we just need to keep the original shape - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - cos, sin = self.rotary_emb(value_states, position_ids) - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - past_key_value = getattr(self, "past_key_value", past_key_value) - - if past_key_value is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - query_states = query_states.transpose(1, 2) - key_states = key_states.transpose(1, 2) - value_states = value_states.transpose(1, 2) - - dropout_rate = self.attention_dropout if self.training else 0.0 - - # In PEFT, usually we cast the layer norms in float32 for training stability reasons - # therefore the input hidden states gets silently casted in float32. Hence, we need - # cast them back in the correct dtype just to be sure everything works as expected. - # This might slowdown training & inference so it is recommended to not cast the LayerNorms - # in fp32. (LlamaRMSNorm handles it correctly) - - input_dtype = query_states.dtype - if input_dtype == torch.float32: - if torch.is_autocast_enabled(): - target_dtype = torch.get_autocast_gpu_dtype() - # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype - else: - target_dtype = self.q_proj.weight.dtype - - logger.warning_once( - f"The input hidden states seems to be silently casted in float32, this might be related to" - f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" - f" {target_dtype}." - ) - - query_states = query_states.to(target_dtype) - key_states = key_states.to(target_dtype) - value_states = value_states.to(target_dtype) - - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attention_mask is not None: # no matter the length, we just slice it - causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] - attn_weights = attn_weights + causal_mask - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) - - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop and past_key_value is not None: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - key_states = key_states * mask.unsqueeze(-1) - value_states = value_states * mask.unsqueeze(-1) - - attn_output = self._flash_attention_forward( - query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate - ) - - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - - def _flash_attention_forward( - self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None - ): - """ - Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token - first unpad the input, then computes the attention scores and pad the final attention scores. - - Args: - query_states (`torch.Tensor`): - Input query states to be passed to Flash Attention API - key_states (`torch.Tensor`): - Input key states to be passed to Flash Attention API - value_states (`torch.Tensor`): - Input value states to be passed to Flash Attention API - attention_mask (`torch.Tensor`): - The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the - position of padding tokens and 1 for the position of non-padding tokens. - dropout (`float`): - Attention dropout - softmax_scale (`float`, *optional*): - The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) - """ - if not self._flash_attn_uses_top_left_mask: - causal = self.is_causal - else: - causal = self.is_causal and query_length != 1 - - # Contains at least one padding token in the sequence - if attention_mask is not None: - batch_size = query_states.shape[0] - query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( - query_states, key_states, value_states, attention_mask, query_length - ) - - cu_seqlens_q, cu_seqlens_k = cu_seq_lens - max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens - - attn_output_unpad = flash_attn_varlen_func( - query_states, - key_states, - value_states, - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_in_batch_q, - max_seqlen_k=max_seqlen_in_batch_k, - dropout_p=dropout, - softmax_scale=softmax_scale, - causal=causal, - ) - - attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) - else: - attn_output = flash_attn_func( - query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal - ) - - return attn_output - - def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): - indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) - batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape - - key_layer = index_first_axis( - key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k - ) - value_layer = index_first_axis( - value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k - ) - if query_length == kv_seq_len: - query_layer = index_first_axis( - query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k - ) - cu_seqlens_q = cu_seqlens_k - max_seqlen_in_batch_q = max_seqlen_in_batch_k - indices_q = indices_k - elif query_length == 1: - max_seqlen_in_batch_q = 1 - cu_seqlens_q = torch.arange( - batch_size + 1, dtype=torch.int32, device=query_layer.device - ) # There is a memcpy here, that is very bad. - indices_q = cu_seqlens_q[:-1] - query_layer = query_layer.squeeze(1) - else: - # The -q_len: slice assumes left padding. - attention_mask = attention_mask[:, -query_length:] - query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) - - return ( - query_layer, - key_layer, - value_layer, - indices_q, - (cu_seqlens_q, cu_seqlens_k), - (max_seqlen_in_batch_q, max_seqlen_in_batch_k), - ) - -class H2OLlamaSdpaAttention(H2OLlamaAttention): - """Llama attention module using torch.nn.functional.scaled_dot_product_attention. - - This module inherits from - `LlamaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to - SDPA API. - """ - - # Adapted from LlamaAttention.forward - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - cache_position: Optional[torch.LongTensor] = None, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - if output_attentions: - logger.warning_once( - "LlamaModel is using LlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` " - "does not support `output_attentions=True`. Falling back to the manual attention implementation, " - 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards.' - '"This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' - ) - return super().forward( - hidden_states=hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_value=past_key_value, - output_attentions=output_attentions, - use_cache=use_cache, - cache_position=cache_position, - ) - - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - try: - cos, sin = self.rotary_emb(value_states, position_ids) # pylint: disable=E1120 - except: - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - kv_seq_len += past_key_value[0].shape[-2] - position_length = kv_seq_len - if not position_ids.nelement() > 1: - if position_length < position_ids.item()+1: - position_length = position_ids.item()+1 - cos, sin = self.rotary_emb(value_states, seq_len=position_length) # pylint: disable=E1120 - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - # In case static cache is used, it is an instance attribute. - past_key_value = getattr(self, "past_key_value", past_key_value) - - if past_key_value is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - causal_mask = attention_mask - if attention_mask is not None: - causal_mask = causal_mask[:, :, :, : key_states.shape[-2]] - - if query_states.device.type == "cuda" and causal_mask is not None: - query_states = query_states.contiguous() - key_states = key_states.contiguous() - value_states = value_states.contiguous() - - # h2o - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attention_mask is not None: # no matter the length, we just slice it - causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] - attn_weights = attn_weights + causal_mask - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop and past_key_value is not None: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - key_states = key_states * mask.unsqueeze(-1) - value_states = value_states * mask.unsqueeze(-1) - - # In case we are not compiling, we may set `causal_mask` to None, - # which is required to dispatch to SDPA's Flash Attention 2 backend, rather - # relying on the `is_causal` argument. - attn_output = torch.nn.functional.scaled_dot_product_attention( - query_states, - key_states, - value_states, - attn_mask=causal_mask, - dropout_p=self.attention_dropout if self.training else 0.0, - is_causal=causal_mask is None and q_len > 1, - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.view(bsz, q_len, self.hidden_size) - - attn_output = self.o_proj(attn_output) - - return attn_output, None, past_key_value From 4c264878c37e4283ffb56908fa9eab09cca570d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 05:57:37 +0000 Subject: [PATCH 34/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../modeling/kv_cache_compression/h2o.py | 2 +- .../kv_cache_compression/models/modeling_llama.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 91d5109d53c..23a5fc781cf 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -252,7 +252,7 @@ def _update_hh_score(self, attn_score_cache, mean=False): attn_score_cache[:, :, :self.hh_score.shape[-1]] += self.hh_score else: attn_score_cache[:,:,:self.hh_score.shape[-1]] = attn_score_cache[:,:,:self.hh_score.shape[-1]] \ - * (self.idx - 1) + self.hh_score + * (self.idx - 1) + self.hh_score attn_score_cache /= self.idx self.hh_score = attn_score_cache diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index e69de29bb2d..2045808acc2 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + From 741c7cdf2d5da0bab4579f7feb5e36497d6a0e01 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 07:09:34 +0000 Subject: [PATCH 35/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../modeling/kv_cache_compression/models/modeling_llama.py | 1 - 1 file changed, 1 deletion(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 2045808acc2..28f108cb636 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - From 0800df236f253698dae030952742bccf1f7c8448 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Sun, 26 May 2024 20:15:24 -0400 Subject: [PATCH 36/62] add example readme Signed-off-by: n1ck-guo --- .../pytorch/text-generation/h2o/README.md | 35 +++++ .../modeling/kv_cache_compression/h2o.py | 18 ++- .../models/modeling_mistral.py | 129 ++++++++++++++++++ .../models/modeling_opt.py | 2 +- 4 files changed, 179 insertions(+), 5 deletions(-) create mode 100644 examples/huggingface/pytorch/text-generation/h2o/README.md diff --git a/examples/huggingface/pytorch/text-generation/h2o/README.md b/examples/huggingface/pytorch/text-generation/h2o/README.md new file mode 100644 index 00000000000..6f198d60b33 --- /dev/null +++ b/examples/huggingface/pytorch/text-generation/h2o/README.md @@ -0,0 +1,35 @@ +# H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models +Code for the paper "**H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models**" + +## Usage and Examples +### Evaluation on tasks from [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness) framework +Using simulation mode +```bash +python run_generation.py \ + --model meta-llama/Meta-Llama-3-8B \ + --accuracy \ + --batch_size 16 \ + --h2o \ + --heavy_ratio 0.1 \ + --recent_ratio 0.1 \ + --device 0 +``` +To run the real_drop mode +```bash +python run_generation.py \ + --model meta-llama/Meta-Llama-3-8B \ + --accuracy \ + --batch_size 16 \ + --h2o \ + --heavy_ratio 0.1 \ + --recent_ratio 0.1 \ + --device 0 + --real_drop +``` +Get the accuracy of dense model +```bash +python run_generation.py \ + --model meta-llama/Meta-Llama-3-8B \ + --accuracy \ + --batch_size 16 +``` \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index 91d5109d53c..195060a67ad 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import importlib +from functools import partial import torch from torch import nn @@ -60,6 +61,14 @@ def clean_cache(model): if "H2O" in module.__class__.__name__: module.h2o_kv_cache.clean_scores() +def generate(model, **kwargs): + for _, module in model.named_modules(): + if "H2O" in module.__class__.__name__: + module.is_gen = True + result = model.ori_generate(**kwargs) + clean_cache(model) + return result + def convert_model( model, heavy_ratio, @@ -91,11 +100,12 @@ def convert_model( recent_ratio, h2o_min_seqlen=h2o_min_seqlen, real_drop=real_drop, - is_gen=is_gen, mean=mean ) set_module(model, layer_name, module) model.clean_cache = lambda: clean_cache(model) + model.ori_generate = model.generate + model.generate = partial(generate, model) model = model.to(device) return model @@ -203,9 +213,8 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): if self.real_drop: return key_states, value_states else: - return torch.ones(attn_score.shape, dtype=attn_score.dtype).to(key_states.device) + return torch.ones(attn_score.shape[:-1], dtype=attn_score.dtype).to(key_states.device) self.idx += 1 - mask_shape = attn_score.shape[:-1] # attn_score shape (bsz, num_heads, seq_len, head_dim) if len(attn_score.shape) == 3: attn_score = attn_score.unsqueeze(0) @@ -222,7 +231,7 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): mask = mask.scatter(-1, keep_topk, 1) if not self.real_drop: - return mask.view(mask_shape) + return mask mask = mask.bool() self.hh_score = self.hh_score[mask].view(self.hh_score.shape[0], self.hh_score.shape[1], cache_size) @@ -241,6 +250,7 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): return k_hh_recent, v_hh_recent def _update_hh_score(self, attn_score_cache, mean=False): + # attn_score_cache (bsz, num_heads, seq_len, head_dim) # hh_score size (bsz, num_heads, head_dim) attn_score_cache = attn_score_cache.sum(-2) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index e912984cb11..ab8d7f0cc93 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -195,3 +195,132 @@ def forward( attn_weights = None return attn_output, attn_weights, past_key_value + +class H2OMistralSdpaAttention(H2OMistralAttention): + """ + Mistral attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + `MistralAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from MistralAttention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "MistralModel is using MistralSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # Reference: https://github.com/pytorch/pytorch/issues/112577. + + + # h2o + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + if past_key_value is not None: + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + # breakpoint() + key_states = key_states * mask.unsqueeze(-1) + value_states = value_states * mask.unsqueeze(-1) + if query_states.device.type == "cuda" and attention_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=attention_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1. + is_causal=self.is_causal and attention_mask is None and q_len > 1, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index e95c2fed7bf..50eb7a5468c 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -193,7 +193,7 @@ def forward( past_key_value[1], mean=self.mean ) - attn_weights = attn_weights * mask.unsqueeze(1) + attn_weights = attn_weights * mask.squeeze(0).unsqueeze(1) # sim # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) From 558dfd9cb78cb94e21620dcbde9910e0df2c6e31 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Sun, 26 May 2024 20:16:14 -0400 Subject: [PATCH 37/62] update Signed-off-by: n1ck-guo --- .../models/modeling_llama.py | 603 ++++++++++++++++++ 1 file changed, 603 insertions(+) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index e69de29bb2d..5b4893ba5ba 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -0,0 +1,603 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch llama model.""" + +import math +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformers.cache_utils import Cache +from transformers.utils import logging +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import rotate_half, repeat_kv, _get_unpad_data + +from ..h2o import H2OKVCache + + +logger = logging.get_logger(__name__) + +from packaging import version +import transformers +if version.parse(transformers.__version__) > version.parse("4.33.0"): + from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available + if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func # pylint: disable=E1101 + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # pylint: disable=E1101 + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`, *optional*): + Deprecated and unused. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + +class H2OLlamaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper.""" + + def __init__( + self, + model, + config: LlamaConfig, + heavy_ratio, + recent_ratio, + h2o_min_seqlen=1024, + real_drop=False, + is_gen=False, + mean=False + ): + super().__init__() + self.config = config + self.layer_idx = model.layer_idx + if self.layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + + self.q_proj = model.q_proj + self.k_proj = model.k_proj + self.v_proj = model.v_proj + self.o_proj = model.o_proj + self.rotary_emb = model.rotary_emb + + # for h2o + self.is_gen = is_gen + self.real_drop = real_drop + self.mean = mean + + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.h2o_min_seqlen = h2o_min_seqlen + + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, # transformers.cache_utils.DynamicCache + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + if self.config.pretraining_tp > 1: + key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp + query_slices = self.q_proj.weight.split( + (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 + ) + key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) + value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) + + query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.config.pretraining_tp)] + query_states = torch.cat(query_states, dim=-1) + + key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.config.pretraining_tp)] + key_states = torch.cat(key_states, dim=-1) + + value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.config.pretraining_tp)] + value_states = torch.cat(value_states, dim=-1) + + else: + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + past_key_value = getattr(self, "past_key_value", past_key_value) + try: # pylint: disable=E1120 + cos, sin = self.rotary_emb(value_states, position_ids) + except: # for old version + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + position_length = kv_seq_len + if not position_ids.nelement() > 1: + if position_length < position_ids.item()+1: + position_length = position_ids.item()+1 + cos, sin = self.rotary_emb(value_states, seq_len=position_length) # pylint: disable=E1120 + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + + # H2O + if past_key_value is not None: + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + attn_weights = attn_weights * mask.unsqueeze(-2) + value_states = value_states * mask.unsqueeze(-1) + # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) + # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + if self.config.pretraining_tp > 1: + attn_output = attn_output.split(self.hidden_size // self.config.pretraining_tp, dim=2) + o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.config.pretraining_tp, dim=1) + attn_output = sum([F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.config.pretraining_tp)]) + else: + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + +class H2OLlamaFlashAttention2(H2OLlamaAttention): + """Llama flash attention module. + + This module inherits from `LlamaAttention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, + # that was made default for flash_attn>=2.1. This attribute is used to handle this difference. + # Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, + # using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + output_attentions = False + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dim x hidden_dim + # therefore we just need to keep the original shape + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + cos, sin = self.rotary_emb(value_states, position_ids) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + past_key_value = getattr(self, "past_key_value", past_key_value) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + dropout_rate = self.attention_dropout if self.training else 0.0 + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in the correct dtype just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. (LlamaRMSNorm handles it correctly) + + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + + if past_key_value is not None: + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + key_states = key_states * mask.unsqueeze(-1) + value_states = value_states * mask.unsqueeze(-1) + + attn_output = self._flash_attention_forward( + query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate + ) + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + def _flash_attention_forward( + self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + """ + if not self._flash_attn_uses_top_left_mask: + causal = self.is_causal + else: + causal = self.is_causal and query_length != 1 + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + attn_output = flash_attn_func( + query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal + ) + + return attn_output + + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape + + key_layer = index_first_axis( + key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + value_layer = index_first_axis( + value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + +class H2OLlamaSdpaAttention(H2OLlamaAttention): + """Llama attention module using torch.nn.functional.scaled_dot_product_attention. + + This module inherits from + `LlamaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from LlamaAttention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + logger.warning_once( + "LlamaModel is using LlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` " + "does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards.' + '"This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + ) + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + try: + cos, sin = self.rotary_emb(value_states, position_ids) # pylint: disable=E1120 + except: + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + position_length = kv_seq_len + if not position_ids.nelement() > 1: + if position_length < position_ids.item()+1: + position_length = position_ids.item()+1 + cos, sin = self.rotary_emb(value_states, seq_len=position_length) # pylint: disable=E1120 + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + # In case static cache is used, it is an instance attribute. + past_key_value = getattr(self, "past_key_value", past_key_value) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + causal_mask = attention_mask + if attention_mask is not None: + causal_mask = causal_mask[:, :, :, : key_states.shape[-2]] + + if query_states.device.type == "cuda" and causal_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + # h2o + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + + if past_key_value is not None: + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + else: + mask = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + # breakpoint() + key_states = key_states * mask.unsqueeze(-1) + value_states = value_states * mask.unsqueeze(-1) + + # In case we are not compiling, we may set `causal_mask` to None, + # which is required to dispatch to SDPA's Flash Attention 2 backend, rather + # relying on the `is_causal` argument. + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=causal_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + is_causal=causal_mask is None and q_len > 1, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value + From 693983f636a3221bda9ccff20c09451db0af7af8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 00:18:08 +0000 Subject: [PATCH 38/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../kv_cache_compression/models/modeling_llama.py | 1 - .../kv_cache_compression/models/modeling_mistral.py | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 5b4893ba5ba..90356342212 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -600,4 +600,3 @@ def forward( attn_output = self.o_proj(attn_output) return attn_output, None, past_key_value - diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index ab8d7f0cc93..64a1c6f8fd8 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -197,8 +197,9 @@ def forward( return attn_output, attn_weights, past_key_value class H2OMistralSdpaAttention(H2OMistralAttention): - """ - Mistral attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + """Mistral attention module using torch.nn.functional.scaled_dot_product_attention. + + This module inherits from `MistralAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to SDPA API. """ @@ -260,7 +261,7 @@ def forward( # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, # Reference: https://github.com/pytorch/pytorch/issues/112577. - + # h2o attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) @@ -323,4 +324,4 @@ def forward( attn_output = self.o_proj(attn_output) - return attn_output, None, past_key_value \ No newline at end of file + return attn_output, None, past_key_value From 30bed259f0dce3eff27dd3d4d78a2d44df35dbec Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Thu, 6 Jun 2024 03:14:15 -0400 Subject: [PATCH 39/62] fix acc bug Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/h2o.py | 83 +++++++++---------- .../models/modeling_bloom.py | 28 ++----- .../models/modeling_gpt_neox.py | 29 ++----- .../models/modeling_llama.py | 35 ++++---- .../models/modeling_mistral.py | 27 +++--- .../models/modeling_mixtral.py | 21 ++--- .../models/modeling_opt.py | 48 +++++------ 7 files changed, 121 insertions(+), 150 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index e1ab5eeb762..f9dffcc07f4 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -20,6 +20,15 @@ import torch from torch import nn +SIM_CLS_MAPPING = { + 'llama': 'H2OLlamaAttention', + 'opt': 'H2OOPTAttention', + 'bloom': 'H2OBloomAttention', + 'mistral': 'H2OMistralAttention', + 'mixtral': 'H2OMixtralAttention', + 'gpt_neox': 'H2OGPTNeoXAttention', +} + def set_module(model, op_name, new_module): """Set module with a given op name. @@ -76,7 +85,8 @@ def convert_model( h2o_min_seqlen=1024, real_drop=True, is_gen=False, - mean=False + mean=False, + local=True ): model_type = model.config.model_type device = model.device @@ -87,12 +97,13 @@ def convert_model( for layer_name in atten_layers: module = get_module(model, layer_name) + cls_name = "H2O" + module.__class__.__name__ if real_drop else SIM_CLS_MAPPING[model_type] h2o_cls = getattr( importlib.import_module( f".models.modeling_{model_type}", "intel_extension_for_transformers.transformers.modeling.kv_cache_compression" ), - "H2O" + module.__class__.__name__) + cls_name) module = h2o_cls( module, model.config, @@ -100,7 +111,9 @@ def convert_model( recent_ratio, h2o_min_seqlen=h2o_min_seqlen, real_drop=real_drop, - mean=mean + is_gen=is_gen, + mean=mean, + local=local ) set_module(model, layer_name, module) model.clean_cache = lambda: clean_cache(model) @@ -112,10 +125,7 @@ def convert_model( def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): - # attn_weights (head, query, keys) or (BS, head, query, keys) - attn_shape_len = len(attn_weights.shape) - assert attn_shape_len in [3,4], \ - "Wrong shape of attn_weights. Should be (head, query, keys) or (BS, head, query, keys)" + # attn_weights (BS, head, query, keys) dtype_attn_weights = attn_weights.dtype seq_length = attn_weights.shape[-1] if no_padding_seq_length is None: @@ -126,41 +136,23 @@ def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=No offset = torch.finfo(attn_weights.dtype).min tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(dtype_attn_weights) - if len(attn_weights.shape) == 3: - accumulated_attention_score = torch.sum( - tmp_attn[:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) - accumulated_attention_score[:,heavy_budget+padding_length:] = 0 - if padding_length > 0: - accumulated_attention_score[:,:padding_length] = 0 - mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - mask_bottom[:,padding_length:heavy_budget+padding_length, - padding_length:heavy_budget+padding_length] = True - else: - accumulated_attention_score = torch.sum( - tmp_attn[:,:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) - accumulated_attention_score[:,:,heavy_budget+padding_length:] = 0 - accumulated_attention_score[:,:,:padding_length] = 0 + accumulated_attention_score = torch.sum( + tmp_attn[:,:,padding_length:heavy_budget+padding_length,:], dim=-2) #(head, keys) + accumulated_attention_score[:,:,heavy_budget+padding_length:] = 0 + accumulated_attention_score[:,:,:padding_length] = 0 - mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) - mask_bottom[:,:, padding_length:heavy_budget+padding_length, - padding_length:heavy_budget+padding_length] = True + mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) + mask_bottom[:,:, padding_length:heavy_budget+padding_length, + padding_length:heavy_budget+padding_length] = True for token_index in range(heavy_budget+padding_length, seq_length): - if attn_shape_len == 3: - tmp_attn_index = nn.functional.softmax( - attn_weights[:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) - else: - tmp_attn_index = nn.functional.softmax( - attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + tmp_attn_index = nn.functional.softmax( + attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) - if attn_shape_len == 3: - mask_bottom_index[:, token_index] = True - mask_bottom[:,token_index,:] = mask_bottom_index - else: - mask_bottom_index[:,:, token_index] = True - mask_bottom[:,:,token_index,:] = mask_bottom_index + mask_bottom_index[:,:, token_index] = True + mask_bottom[:,:,token_index,:] = mask_bottom_index accumulated_attention_score += tmp_attn_index accumulated_attention_score = accumulated_attention_score * mask_bottom_index @@ -169,12 +161,21 @@ def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=No return mask_bottom -def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights): +def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights, local=True): heavy_budget = int(heavy_budget_ratio * attn_weights.shape[-1]) recent_budget = int(recent_budget_ratio * attn_weights.shape[-1]) if heavy_budget > 0: # Default: No padding applied to input - mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) + if local: + mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) + else: + tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(attn_weights.dtype) + tmp_sum = torch.sum(tmp_attn, dim=-2) + _, tmp_topk = tmp_sum.topk(k=heavy_budget, dim=-1) + + zeros = torch.zeros_like(tmp_sum, dtype=torch.bool) + mask_bottom = zeros.scatter(-1, tmp_topk, True).unsqueeze(2) + mask_bottom = mask_bottom.expand(mask_bottom.shape[0], mask_bottom.shape[1], attn_weights.shape[-2], mask_bottom.shape[-1]) else: mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) @@ -192,14 +193,13 @@ def __init__( self, heavy_ratio=0.2, recent_ratio=0.2, - real_drop=False, + # real_drop=False, min_seqlen=-1 ): ## bsz, num_heads, seq_len, head_dim self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.hh_score = None - self.real_drop = real_drop self.min_seqlen = min_seqlen self.idx = 0 @@ -230,9 +230,6 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): _, keep_topk = torch.topk(select_hh_scores, heavy_budget, dim=-1, largest=True) mask = mask.scatter(-1, keep_topk, 1) - if not self.real_drop: - return mask - mask = mask.bool() self.hh_score = self.hh_score[mask].view(self.hh_score.shape[0], self.hh_score.shape[1], cache_size) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py index ec82b5de563..d61b312931e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -37,7 +37,8 @@ def __init__( h2o_min_seqlen=1024, real_drop=False, is_gen=False, - mean=False + mean=False, + local=True ): super().__init__() @@ -71,6 +72,7 @@ def __init__( self.real_drop = real_drop self.is_gen = is_gen self.mean = mean + self.local = local self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio @@ -171,28 +173,12 @@ def forward( attention_scores = attention_scores.to(torch.float) attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.finfo(attention_scores.dtype).min) - attention_probs = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(input_dtype) - # get hh mask - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attention_probs, - key_layer, - value_layer, - mean=self.mean - ) - past_key_value = (new_key_states, new_value_states) - else: - mask = self.h2o_kv_cache( - attention_probs, - key_layer, - value_layer, - mean=self.mean - ) - attention_probs = attention_probs * mask.unsqueeze(-2) + from ..h2o import get_hh_mask + mask = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights.detach().clone(), local=self.local) + attn_weights[~mask] = torch.finfo(attn_weights.dtype).min + attention_probs = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(input_dtype) # [batch_size, num_heads, q_length, kv_length] attention_probs = self.attention_dropout(attention_probs) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py index bc0110f54f8..840095ec62a 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -24,7 +24,7 @@ logger = logging.get_logger(__name__) -class GPTNeoXAttention(nn.Module): +class H2OGPTNeoXAttention(nn.Module): def __init__( self, model, @@ -34,7 +34,8 @@ def __init__( h2o_min_seqlen=1024, real_drop=False, is_gen=False, - mean=False + mean=False, + local=True ): super().__init__() self.config = config @@ -60,10 +61,11 @@ def __init__( # for h2o if real_drop: real_drop = False - logger.warning_once("BloomAttention not support for kv cache, usning simulation mode.") + logger.warning_once("GPTNeoXAttention not support for kv cache, usning simulation mode.") self.real_drop = real_drop self.is_gen = is_gen self.mean = mean + self.local = local self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio @@ -199,24 +201,9 @@ def _attn(self, query, key, value, attention_mask=None, head_mask=None): attn_weights = attn_weights.to(value.dtype) # get hh mask - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - key, - value, - mean=self.mean - ) - past_key_value = (new_key_states, new_value_states) - else: - mask = self.h2o_kv_cache( - attn_weights, - key, - value, - mean=self.mean - ) - attn_weights = attn_weights * mask.unsqueeze(1) + from ..h2o import get_hh_mask + mask = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights.detach().clone(), local=self.local) + attn_weights[~mask] = torch.finfo(attn_weights.dtype).min # Mask heads if we want to if head_mask is not None: diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 5b4893ba5ba..b223b3def09 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -80,7 +80,8 @@ def __init__( h2o_min_seqlen=1024, real_drop=False, is_gen=False, - mean=False + mean=False, + local=True ): super().__init__() self.config = config @@ -118,12 +119,13 @@ def __init__( self.is_gen = is_gen self.real_drop = real_drop self.mean = mean + self.local = local self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.h2o_min_seqlen = h2o_min_seqlen - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_min_seqlen) def forward( self, @@ -190,17 +192,13 @@ def forward( causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] attn_weights = attn_weights + causal_mask - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) - # H2O if past_key_value is not None: if not self.is_gen: self.h2o_kv_cache.clean_scores() if self.real_drop: new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, + attn_weights.detach().clone(), past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], mean=self.mean @@ -208,16 +206,17 @@ def forward( past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - attn_weights = attn_weights * mask.unsqueeze(-2) - value_states = value_states * mask.unsqueeze(-1) - # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + from ..h2o import get_hh_mask + mask = get_hh_mask( + self.heavy_ratio, + self.recent_ratio, + attn_weights.detach().clone(), + local=self.local) + attn_weights[~mask] = torch.finfo(attn_weights.dtype).min + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) @@ -557,7 +556,7 @@ def forward( attn_weights = attn_weights + causal_mask # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + # attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) if past_key_value is not None: if not self.is_gen: diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index ab8d7f0cc93..9ee078cf2ee 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -44,7 +44,8 @@ def __init__( h2o_min_seqlen=1024, real_drop=False, is_gen=False, - mean=False + mean=False, + local=True ): super().__init__() self.config = config @@ -81,6 +82,7 @@ def __init__( self.is_gen = is_gen self.real_drop = real_drop self.mean = mean + self.local = local self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio @@ -152,15 +154,12 @@ def forward( attn_weights = attn_weights + attention_mask - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - # H2O if not self.is_gen: self.h2o_kv_cache.clean_scores() if self.real_drop and past_key_value is not None: new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, + attn_weights.detach().clone(), past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], mean=self.mean @@ -168,16 +167,18 @@ def forward( past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - attn_weights = attn_weights * mask.unsqueeze(-2) - value_states = value_states * mask.unsqueeze(-1) + from ..h2o import get_hh_mask + mask = get_hh_mask( + self.heavy_ratio, + self.recent_ratio, + attn_weights.detach().clone(), + local=self.local) + attn_weights[~mask] = torch.finfo(attn_weights.dtype).min + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index cc0a82019f7..2da2ae6274e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -44,7 +44,8 @@ def __init__( h2o_min_seqlen=1024, real_drop=False, is_gen=False, - mean=False + mean=False, + local=True ): super().__init__() self.config = config @@ -82,6 +83,7 @@ def __init__( self.is_gen = is_gen self.real_drop = real_drop self.mean = mean + self.local = local self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio @@ -161,7 +163,7 @@ def forward( self.h2o_kv_cache.clean_scores() if self.real_drop and past_key_value is not None: new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, + attn_weights.detach().clone(), past_key_value.key_cache[self.layer_idx], past_key_value.value_cache[self.layer_idx], mean=self.mean @@ -169,14 +171,13 @@ def forward( past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - attn_weights = attn_weights * mask.unsqueeze(-2) - value_states = value_states * mask.unsqueeze(-1) + from ..h2o import get_hh_mask + mask = get_hh_mask( + self.heavy_ratio, + self.recent_ratio, + attn_weights.detach().clone(), + local=self.local) + attn_weights[~mask] = torch.finfo(attn_weights.dtype).min attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index 50eb7a5468c..ff5ef8ace53 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -37,7 +37,8 @@ def __init__( h2o_min_seqlen=1024, real_drop=False, is_gen=False, - mean=False + mean=False, + local=True ): super().__init__() self.config = config @@ -66,6 +67,7 @@ def __init__( self.is_gen = is_gen self.real_drop = real_drop self.mean = mean + self.local = local self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio @@ -150,6 +152,26 @@ def forward( ) attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + # get hh mask + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + if self.real_drop: + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights.detach().clone(), + past_key_value[0], + past_key_value[1], + mean=self.mean + ) + past_key_value = (new_key_states, new_value_states) + else: + from ..h2o import get_hh_mask + mask = get_hh_mask( + self.heavy_ratio, + self.recent_ratio, + attn_weights.detach().clone(), + local=self.local) + attn_weights[~mask.squeeze(0)] = torch.finfo(attn_weights.dtype).min + # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 if attn_weights.dtype == torch.float16: attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(torch.float16) @@ -175,29 +197,7 @@ def forward( else: attn_weights_reshaped = None - # get hh mask - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - mean=self.mean - ) - past_key_value = (new_key_states, new_value_states) - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - mean=self.mean - ) - attn_weights = attn_weights * mask.squeeze(0).unsqueeze(1) - - # sim - # mask_bottom = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights) - # attn_weights[~mask_bottom] = torch.finfo(attn_weights.dtype).min + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) From f8a64fcd57d5950328fd66db98652a4e7affffa2 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Thu, 6 Jun 2024 22:08:07 -0400 Subject: [PATCH 40/62] fix Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/h2o.py | 6 +- .../models/modeling_llama.py | 65 +-- .../models/modeling_mistral.py | 49 +- .../models/modeling_mixtral.py | 442 +++++++++++++++++- 4 files changed, 482 insertions(+), 80 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index f9dffcc07f4..b90b2a52490 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -193,7 +193,6 @@ def __init__( self, heavy_ratio=0.2, recent_ratio=0.2, - # real_drop=False, min_seqlen=-1 ): ## bsz, num_heads, seq_len, head_dim @@ -210,10 +209,7 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): recent_budget = int(self.recent_ratio * seq_len) cache_size = heavy_budget + recent_budget if seq_len <= self.min_seqlen or seq_len <= cache_size: - if self.real_drop: - return key_states, value_states - else: - return torch.ones(attn_score.shape[:-1], dtype=attn_score.dtype).to(key_states.device) + return torch.ones(attn_score.shape[:-1], dtype=attn_score.dtype).to(key_states.device) self.idx += 1 # attn_score shape (bsz, num_heads, seq_len, head_dim) if len(attn_score.shape) == 3: diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index b223b3def09..236ec5d4058 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -38,8 +38,13 @@ if version.parse(transformers.__version__) > version.parse("4.33.0"): from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available if is_flash_attn_2_available(): - from flash_attn import flash_attn_func, flash_attn_varlen_func # pylint: disable=E1101 - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # pylint: disable=E1101 + from flash_attn import ( + flash_attn_func, + flash_attn_varlen_func) # pylint: disable=E1101 + from flash_attn.bert_padding import ( + index_first_axis, + pad_input, + unpad_input) # pylint: disable=E1101 def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): @@ -335,31 +340,19 @@ def forward( causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] attn_weights = attn_weights + causal_mask - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + # h2o if past_key_value is not None: if not self.is_gen: self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean ) - key_states = key_states * mask.unsqueeze(-1) - value_states = value_states * mask.unsqueeze(-1) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states attn_output = self._flash_attention_forward( query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate @@ -555,31 +548,17 @@ def forward( causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] attn_weights = attn_weights + causal_mask - # upcast attention to fp32 - # attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - if past_key_value is not None: if not self.is_gen: self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean ) - # breakpoint() - key_states = key_states * mask.unsqueeze(-1) - value_states = value_states * mask.unsqueeze(-1) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states # In case we are not compiling, we may set `causal_mask` to None, # which is required to dispatch to SDPA's Flash Attention 2 backend, rather diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index 9ee078cf2ee..84128e3164e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -199,9 +199,9 @@ def forward( class H2OMistralSdpaAttention(H2OMistralAttention): """ - Mistral attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from - `MistralAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to - SDPA API. + Mistral attention module using torch.nn.functional.scaled_dot_product_attention. + This module inherits from `MistralAttention` as the weights of the module stays untouched. + The only changes are on the forward pass to adapt to SDPA API. """ # Adapted from MistralAttention.forward @@ -217,8 +217,10 @@ def forward( if output_attentions: # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. logger.warning_once( - "MistralModel is using MistralSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " - 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + "MistralModel is using MistralSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention`" + "does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards.' + ' This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' ) return super().forward( hidden_states=hidden_states, @@ -259,7 +261,8 @@ def forward( f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" ) - # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged + # with non-contiguous inputs with custom attn_mask, # Reference: https://github.com/pytorch/pytorch/issues/112577. @@ -275,35 +278,24 @@ def forward( if attention_mask is not None: if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, \ + but is {attention_mask.size()}" ) attn_weights = attn_weights + attention_mask - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) if past_key_value is not None: if not self.is_gen: self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean ) - # breakpoint() - key_states = key_states * mask.unsqueeze(-1) - value_states = value_states * mask.unsqueeze(-1) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + if query_states.device.type == "cuda" and attention_mask is not None: query_states = query_states.contiguous() key_states = key_states.contiguous() @@ -315,7 +307,8 @@ def forward( value_states, attn_mask=attention_mask, dropout_p=self.attention_dropout if self.training else 0.0, - # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1. + # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d + # that does not create a causal mask in case q_len == 1. is_causal=self.is_causal and attention_mask is None and q_len > 1, ) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index 2da2ae6274e..3c0fd4f7e1e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -14,6 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import inspect import math import logging from typing import List, Optional, Tuple, Union @@ -22,13 +23,20 @@ import torch.nn as nn from transformers.cache_utils import Cache -from transformers.models.mixtral.modeling_mixtral import apply_rotary_pos_emb, repeat_kv +from transformers.models.mixtral.modeling_mixtral import apply_rotary_pos_emb, repeat_kv, _get_unpad_data +from transformers.utils import is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10 from ..h2o import H2OKVCache logger = logging.getLogger(__name__) -class MixtralAttention(nn.Module): +if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) + +class H2OMixtralAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer @@ -155,8 +163,6 @@ def forward( attn_weights = attn_weights + attention_mask - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) # H2O if not self.is_gen: @@ -179,6 +185,8 @@ def forward( local=self.local) attn_weights[~mask] = torch.finfo(attn_weights.dtype).min + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) @@ -197,3 +205,429 @@ def forward( attn_weights = None return attn_output, attn_weights, past_key_value + +class H2OMixtralFlashAttention2(H2OMixtralAttention): + """ + Mixtral flash attention module. This module inherits from `MixtralAttention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, + # that was made default for flash_attn>=2.1. This attribute is used to handle this difference. + # Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + ): + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + + # Because the input can be padded, the absolute sequence length depends on the max position id. + rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1 + cos, sin = self.rotary_emb(value_states, seq_len=rotary_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + use_sliding_windows = ( + _flash_supports_window_size + and getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + ) + + if not _flash_supports_window_size: + logger.warning_once( + "The current flash attention version does not support sliding window attention," + "for a more memory efficient implementation" + " make sure to upgrade flash-attn library." + ) + + if past_key_value is not None: + # Activate slicing cache only if the config has a value `sliding_windows` attribute + cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 + if ( + getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + and cache_has_contents + ): + slicing_tokens = 1 - self.config.sliding_window + + past_key = past_key_value[self.layer_idx][0] + past_value = past_key_value[self.layer_idx][1] + + past_key = past_key[:, :, slicing_tokens:, :].contiguous() + past_value = past_value[:, :, slicing_tokens:, :].contiguous() + + if past_key.shape[-2] != self.config.sliding_window - 1: + raise ValueError( + f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" + f" {past_key.shape}" + ) + + if attention_mask is not None: + attention_mask = attention_mask[:, slicing_tokens:] + attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) + + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + dropout_rate = 0.0 if not self.training else self.attention_dropout + + # h2o + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + if past_key_value is not None: + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 just to be sure everything works as expected. + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + # Reashape to the expected shape for Flash Attention + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + attn_output = self._flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + q_len, + dropout=dropout_rate, + use_sliding_windows=use_sliding_windows, + ) + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + def _flash_attention_forward( + self, + query_states, + key_states, + value_states, + attention_mask, + query_length, + dropout=0.0, + softmax_scale=None, + use_sliding_windows=False, + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + use_sliding_windows (`bool`, *optional*): + Whether to activate sliding window attention. + """ + if not self._flash_attn_uses_top_left_mask: + causal = self.is_causal + else: + # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. + causal = self.is_causal and query_length != 1 + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + if not use_sliding_windows: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + if not use_sliding_windows: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + return attn_output + + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape + + # On the first iteration we need to properly re-create the padding mask + # by slicing it on the proper place + if kv_seq_len != attention_mask.shape[-1]: + attention_mask_num_tokens = attention_mask.shape[-1] + attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :] + + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + + key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + +class H2OMixtralSdpaAttention(H2OMixtralAttention): + """ + Mixtral attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + `MixtralAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from MixtralAttention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "MixtralModel is using MixtralSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention`" + "does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards.' + 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + # h2o + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + if past_key_value is not None: + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # Reference: https://github.com/pytorch/pytorch/issues/112577. + if query_states.device.type == "cuda" and attention_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=attention_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1. + is_causal=self.is_causal and attention_mask is None and q_len > 1, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value \ No newline at end of file From 7a12ec6548db4a563b09ecd06861a1ad8c945586 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 00:54:21 +0000 Subject: [PATCH 41/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../modeling/kv_cache_compression/h2o.py | 2 +- .../models/modeling_mistral.py | 4 ++-- .../models/modeling_mixtral.py | 14 ++++++++------ .../kv_cache_compression/models/modeling_opt.py | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index b90b2a52490..d625e3f7cfb 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -170,7 +170,7 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights, local=Tru mask_bottom = local_heavy_hitter_mask(attn_weights, heavy_budget, None) else: tmp_attn = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(attn_weights.dtype) - tmp_sum = torch.sum(tmp_attn, dim=-2) + tmp_sum = torch.sum(tmp_attn, dim=-2) _, tmp_topk = tmp_sum.topk(k=heavy_budget, dim=-1) zeros = torch.zeros_like(tmp_sum, dtype=torch.bool) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index 6a5e9a4a9e1..2ef64ed5437 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -198,8 +198,8 @@ def forward( return attn_output, attn_weights, past_key_value class H2OMistralSdpaAttention(H2OMistralAttention): - """ - Mistral attention module using torch.nn.functional.scaled_dot_product_attention. + """Mistral attention module using torch.nn.functional.scaled_dot_product_attention. + This module inherits from `MistralAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to SDPA API. """ diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index 3c0fd4f7e1e..e6d8aeb93f8 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -207,8 +207,9 @@ def forward( return attn_output, attn_weights, past_key_value class H2OMixtralFlashAttention2(H2OMixtralAttention): - """ - Mixtral flash attention module. This module inherits from `MixtralAttention` as the weights of the module stays + """Mixtral flash attention module. + + This module inherits from `MixtralAttention` as the weights of the module stays untouched. The only required change would be on the forward pass where it needs to correctly call the public API of flash attention and deal with padding tokens in case the input contains any of them. """ @@ -218,7 +219,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, # that was made default for flash_attn>=2.1. This attribute is used to handle this difference. # Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). @@ -520,8 +521,9 @@ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query ) class H2OMixtralSdpaAttention(H2OMixtralAttention): - """ - Mixtral attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + """Mixtral attention module using torch.nn.functional.scaled_dot_product_attention. + + This module inherits from `MixtralAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to SDPA API. """ @@ -630,4 +632,4 @@ def forward( attn_output = self.o_proj(attn_output) - return attn_output, None, past_key_value \ No newline at end of file + return attn_output, None, past_key_value diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index ff5ef8ace53..d9ad7703ba9 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -197,7 +197,7 @@ def forward( else: attn_weights_reshaped = None - + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) From 124bb72a3ca1a1069b92d17c239807e36293e0f4 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 18 Jun 2024 02:07:52 -0400 Subject: [PATCH 42/62] refactor code Signed-off-by: n1ck-guo --- .../text-generation/h2o/run_generation.py | 79 ++- .../modeling/kv_cache_compression/__init__.py | 8 +- .../modeling/kv_cache_compression/h2o.py | 111 ++-- .../models/modeling_bloom.py | 223 ++++++- .../models/modeling_gpt_neox.py | 485 +++++++++++++- .../models/modeling_llama.py | 426 +++++++++++- .../models/modeling_mistral.py | 615 +++++++++++++++++- .../models/modeling_mixtral.py | 318 ++++++++- .../models/modeling_opt.py | 305 +++++++-- 9 files changed, 2311 insertions(+), 259 deletions(-) diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py index b41712d9d96..94ba87dd634 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py @@ -1,6 +1,5 @@ import argparse import sys -sys.path.insert(0, '/home/hengguo/code/intel-extension-for-transformers') import time import json import torch @@ -66,7 +65,6 @@ # transformers version >= 4.32.0 contained the mpt modeling definition. # https://github.com/huggingface/transformers/blob/main/src/transformers/models/mpt/modeling_mpt.py # 4.31.0 for ipex.optimize_transformers -check_min_version("4.35.2") # get model config if args.peft_model_id: from peft import PeftConfig @@ -111,24 +109,26 @@ device = args.device else: device = f"cuda:{args.device}" -user_model = AutoModelForCausalLM.from_pretrained(args.model, trust_remote_code=args.trust_remote_code) -user_model.to(device) # get optimized model if args.h2o: print('Enable Small Cache Size') - # checkpoint = copy.deepcopy(model.state_dict()) - # model = ENABLE_Heavy_Hitter_FUNCTIONS[args.model_type](model, config) - from intel_extension_for_transformers.transformers.modeling.kv_cache_compression import convert_model - user_model = convert_model( - user_model, + from intel_extension_for_transformers.transformers.modeling.kv_cache_compression import H2OConfig, H2OLlamaForCausalLM + h2o_config = H2OConfig( heavy_ratio=args.heavy_ratio, recent_ratio=args.recent_ratio, h2o_min_seqlen=args.h2o_min_seqlen, real_drop=args.real_drop, - is_gen=args.is_gen - ) + mean=False, + ) + user_model = H2OLlamaForCausalLM.from_pretrained( + args.model, + h2o_config=h2o_config, + trust_remote_code=args.trust_remote_code) print("converted model: ", user_model) +else: + user_model = AutoModelForCausalLM.from_pretrained(args.model, trust_remote_code=args.trust_remote_code) +user_model.to(device) # save model # if args.output_dir is not None: @@ -194,18 +194,45 @@ if args.accuracy: user_model = (user_model.eval() if (not (args.int8 or args.int8_bf16_mixed) and hasattr(user_model, "eval")) \ else user_model) - from intel_extension_for_transformers.transformers.llm.evaluation.lm_eval import evaluate, LMEvalParser - model_args="pretrained="+args.model+",trust_remote_code="+str(args.trust_remote_code) - args.tasks = ",".join(args.tasks) - tokenizer.pad_token = tokenizer.eos_token - eval_args = LMEvalParser(model = "hf", - user_model=user_model, - tokenizer=tokenizer, - model_args=model_args, - tasks = args.tasks, - device = device, - num_fewshot=args.num_fewshot, - output_path=args.save_accuracy_path, - batch_size = args.batch_size) - print("using device:", device) - results = evaluate(eval_args) \ No newline at end of file + # from intel_extension_for_transformers.transformers.llm.evaluation.lm_eval import evaluate, LMEvalParser + # model_args="pretrained="+args.model+",trust_remote_code="+str(args.trust_remote_code) + # args.tasks = ",".join(args.tasks) + # tokenizer.pad_token = tokenizer.eos_token + # eval_args = LMEvalParser(model = "hf", + # user_model=user_model, + # tokenizer=tokenizer, + # model_args=model_args, + # tasks = args.tasks, + # device = device, + # num_fewshot=args.num_fewshot, + # output_path=args.save_accuracy_path, + # batch_size = args.batch_size) + # print("using device:", device) + # results = evaluate(eval_args) + + + # original lm_eval + from lm_eval.evaluator import simple_evaluate + from lm_eval.tasks import TaskManager + import lm_eval + + verbosity = 'INFO' + task_manager = TaskManager(verbosity) + limit = None + cache_requests = False + lm = lm_eval.api.registry.get_model("hf")( + pretrained=user_model, + batch_size=args.batch_size, + max_batch_size=None, + ) + model_args="pretrained="+ args.model+ ",tokenizer="+ args.model + ",dtype=float32" + use_cache = None + results = simple_evaluate( + model=lm, + model_args=model_args, + tasks=args.tasks, + num_fewshot=args.num_fewshot, + device=device + ) + import pprint + pprint.pprint(results["results"]) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py index 99ca690b652..471a68ace95 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py @@ -15,4 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .h2o import convert_model +from .h2o import H2OConfig +from .models.modeling_llama import H2OLlamaForCausalLM +from .models.modeling_bloom import H2OBloomForCausalLM +from .models.modeling_gpt_neox import H2OGPTNeoXForCausalLM +from .models.modeling_opt import H2OOPTForCausalLM +from .models.modeling_mistral import H2OMistralForCausalLM +from .models.modeling_mixtral import H2OMixtralForCausalLM diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index b90b2a52490..2c15ab3725e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -67,61 +67,22 @@ def get_module(model, op_name): def clean_cache(model): for _, module in model.named_modules(): - if "H2O" in module.__class__.__name__: + if "Attention" in module.__class__.__name__: module.h2o_kv_cache.clean_scores() def generate(model, **kwargs): + max_length = kwargs['max_new_tokens'] if kwargs.get('max_new_tokens') else kwargs['max_length'] for _, module in model.named_modules(): - if "H2O" in module.__class__.__name__: + if "Attention" in module.__class__.__name__: module.is_gen = True + if module.h2o_kv_cache.heavy_budget is None: + module.h2o_kv_cache.heavy_budget = int(max_length * module.h2o_kv_cache.heavy_ratio) + if module.h2o_kv_cache.recent_budget is None: + module.h2o_kv_cache.recent_budget = int(max_length * module.h2o_kv_cache.recent_ratio) result = model.ori_generate(**kwargs) clean_cache(model) return result -def convert_model( - model, - heavy_ratio, - recent_ratio, - h2o_min_seqlen=1024, - real_drop=True, - is_gen=False, - mean=False, - local=True - ): - model_type = model.config.model_type - device = model.device - atten_layers = [] - for name, module in model.named_modules(): - if "Attention" in module.__class__.__name__: - atten_layers.append(name) - - for layer_name in atten_layers: - module = get_module(model, layer_name) - cls_name = "H2O" + module.__class__.__name__ if real_drop else SIM_CLS_MAPPING[model_type] - h2o_cls = getattr( - importlib.import_module( - f".models.modeling_{model_type}", - "intel_extension_for_transformers.transformers.modeling.kv_cache_compression" - ), - cls_name) - module = h2o_cls( - module, - model.config, - heavy_ratio, - recent_ratio, - h2o_min_seqlen=h2o_min_seqlen, - real_drop=real_drop, - is_gen=is_gen, - mean=mean, - local=local - ) - set_module(model, layer_name, module) - model.clean_cache = lambda: clean_cache(model) - model.ori_generate = model.generate - model.generate = partial(generate, model) - model = model.to(device) - return model - def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): @@ -188,28 +149,37 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights, local=Tru return mask_bottom + class H2OKVCache: def __init__( self, heavy_ratio=0.2, recent_ratio=0.2, + heavy_budget=None, + recent_budget=None, min_seqlen=-1 ): ## bsz, num_heads, seq_len, head_dim self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio + self.heavy_budget = heavy_budget + self.recent_budget = recent_budget self.hh_score = None self.min_seqlen = min_seqlen self.idx = 0 + + self._past_length = 0 def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): seq_len = key_states.size(-2) - heavy_budget = int(self.heavy_ratio * seq_len) - recent_budget = int(self.recent_ratio * seq_len) - cache_size = heavy_budget + recent_budget + if self.heavy_budget is None: + self.heavy_budget = int(self.heavy_ratio * seq_len) + if self.recent_budget is None: + self.recent_budget = int(self.recent_ratio * seq_len) + cache_size = self.heavy_budget + self.recent_budget if seq_len <= self.min_seqlen or seq_len <= cache_size: - return torch.ones(attn_score.shape[:-1], dtype=attn_score.dtype).to(key_states.device) + return key_states, value_states self.idx += 1 # attn_score shape (bsz, num_heads, seq_len, head_dim) if len(attn_score.shape) == 3: @@ -218,12 +188,12 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): # hh-selection mask = torch.zeros(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) - if not recent_budget == 0: - mask[:,:,-recent_budget:] = 1 - select_hh_scores = self.hh_score[:,:,:seq_len - recent_budget] + if not self.recent_budget == 0: + mask[:,:,-self.recent_budget:] = 1 + select_hh_scores = self.hh_score[:,:,:seq_len - self.recent_budget] - if not heavy_budget == 0: - _, keep_topk = torch.topk(select_hh_scores, heavy_budget, dim=-1, largest=True) + if not self.heavy_budget == 0: + _, keep_topk = torch.topk(select_hh_scores, self.heavy_budget, dim=-1, largest=True) mask = mask.scatter(-1, keep_topk, 1) mask = mask.bool() @@ -260,7 +230,36 @@ def _update_hh_score(self, attn_score_cache, mean=False): self.hh_score = attn_score_cache - def clean_scores(self): self.idx = 0 + self.past_length = 0 self.hh_score = None + + @property + def past_length(self): + return self._past_length + + @past_length.setter + def past_length(self, value): + self._past_length = value + +class H2OConfig(dict): + def __init__( + self, + heavy_ratio: float = None, + recent_ratio: float = None, + heavy_budget: int = None, + recent_budget: int = None, + h2o_min_seqlen: int = -1, + real_drop: bool = True, + mean: bool = False, + local: bool = True + ): + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.heavy_budget = heavy_budget + self.recent_budget = recent_budget + self.h2o_min_seqlen = h2o_min_seqlen + self.real_drop = real_drop + self.mean = mean + self.local = local diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py index d61b312931e..2494208d2d2 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -15,30 +15,38 @@ # See the License for the specific language governing permissions and # limitations under the License. import math +import warnings from torch.nn import functional as F from typing import List, Optional, Tuple, Union import torch import torch.nn as nn - -from ..h2o import H2OKVCache +from torch.nn import CrossEntropyLoss from transformers.utils import logging +from transformers.models.bloom import ( + BloomConfig, + BloomModel, + BloomPreTrainedModel, + BLOOM_INPUTS_DOCSTRING, + _CHECKPOINT_FOR_DOC, + _CONFIG_FOR_DOC +) +from transformers.file_utils import ( + add_code_sample_docstrings, + add_start_docstrings_to_model_forward + ) +from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions + +from ..h2o import H2OKVCache, H2OConfig logger = logging.get_logger(__name__) class H2OBloomAttention(nn.Module): def __init__( self, - model, - config, - heavy_ratio, - recent_ratio, - h2o_min_seqlen=1024, - real_drop=False, - is_gen=False, - mean=False, - local=True + config: BloomConfig, + h2o_config: H2OConfig ): super().__init__() @@ -61,24 +69,24 @@ def __init__( self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim) self.beta = 1.0 - self.query_key_value = model.query_key_value - self.dense = model.dense - self.attention_dropout = model.attention_dropout + self.query_key_value = nn.Linear(self.hidden_size, 3 * self.hidden_size, bias=True) + self.dense = nn.Linear(self.hidden_size, self.hidden_size) + self.attention_dropout = nn.Dropout(config.attention_dropout) # for h2o if real_drop: real_drop = False logger.warning_once("BloomAttention not support for kv cache, usning simulation mode.") - self.real_drop = real_drop - self.is_gen = is_gen - self.mean = mean - self.local = local + self.h2o_config = h2o_config + self.is_gen = False + self.mean = h2o_config.mean + self.local = h2o_config.local - self.heavy_ratio = heavy_ratio - self.recent_ratio = recent_ratio - self.h2o_min_seqlen = h2o_min_seqlen + self.heavy_ratio = h2o_config.heavy_ratio + self.recent_ratio = h2o_config.recent_ratio + self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory @@ -230,3 +238,174 @@ def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: out = F.dropout(x, p=prob, training=training) out = residual + out return out + +class H2OBloomForCausalLM(BloomPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + def __init__( + self, + config: BloomConfig, + h2o_config: H2OConfig + ): + super().__init__(config) + self.transformer = BloomModel(config) + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + num_layers = len(self.transformer.h) + for layer_idx in range(num_layers): + self.transformer.h[layer_idx].self_attention = H2OBloomAttention( + config, + h2o_config + ) + # Initialize weights and apply final processing + self.post_init() + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings: torch.Tensor): + self.lm_head = new_embeddings + + def prepare_inputs_for_generation( + self, + input_ids: torch.LongTensor, + past_key_values: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + **kwargs, + ) -> dict: + # only last tokens for input_ids if past is not None + if past_key_values is not None: + past_length = past_key_values[0][0].shape[2] + + # Some generation methods already pass only the last input ID + if input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = input_ids.shape[1] - 1 + + input_ids = input_ids[:, remove_prefix_length:] + + # the cache may be in the stardard format (e.g. in contrastive search), convert to bloom's format if needed + if past_key_values[0][0].shape[0] == input_ids.shape[0]: + past_key_values = self._convert_to_bloom_cache(past_key_values) + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=CausalLMOutputWithCrossAttentions, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **deprecated_arguments, + ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set + `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` + are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + """ + if deprecated_arguments.pop("position_ids", False) is not False: + # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` + warnings.warn( + "`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore" + " passing `position_ids`.", + FutureWarning, + ) + if len(deprecated_arguments) > 0: + raise ValueError(f"Got unexpected arguments: {deprecated_arguments}") + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.transformer( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + + lm_logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + # move labels to correct device to enable model parallelism + labels = labels.to(lm_logits.device) + # Shift so that tokens < n predict n + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + batch_size, seq_length, vocab_size = shift_logits.shape + # Flatten the tokens + loss_fct = CrossEntropyLoss() + loss = loss_fct( + shift_logits.view(batch_size * seq_length, vocab_size), shift_labels.view(batch_size * seq_length) + ) + + if not return_dict: + output = (lm_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=loss, + logits=lm_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) + + def _reorder_cache( + self, past: Tuple[Tuple[torch.Tensor, torch.Tensor], ...], beam_idx: torch.LongTensor + ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]: + """ + This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or + [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct + beam_idx at every generation step. + + Output shares the same memory storage as `past`. + """ + standardized_past = self._convert_to_standard_cache(past, batch_size=len(beam_idx)) + + # Get a copy of `beam_idx` on all the devices where we need those indices. + device_to_beam_idx = { + past_state.device: beam_idx.to(past_state.device) for layer_past in past for past_state in layer_past + } + reordered_past = tuple( + ( + layer_past[0].index_select(0, device_to_beam_idx[layer_past[0].device]), + layer_past[1].index_select(0, device_to_beam_idx[layer_past[0].device]), + ) + for layer_past in standardized_past + ) + return self._convert_to_bloom_cache(reordered_past) \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py index 840095ec62a..3aa13dbca05 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -18,60 +18,81 @@ import torch import torch.nn as nn +from torch.nn import CrossEntropyLoss from transformers.utils import logging - -from ..h2o import H2OKVCache +from transformers.models.gpt_neox.modeling_gpt_neox import ( + _get_unpad_data, + GPTNeoXPreTrainedModel, + GPTNeoXModel, + CausalLMOutputWithPast, + GPT_NEOX_INPUTS_DOCSTRING, + GPT_NEOX_START_DOCSTRING, + _CONFIG_FOR_DOC + +) +from transformers.file_utils import ( + add_start_docstrings, + replace_return_docstrings, + add_start_docstrings_to_model_forward, + ) + +from ..h2o import H2OKVCache, H2OConfig logger = logging.get_logger(__name__) +from packaging import version +import transformers +if version.parse(transformers.__version__) > version.parse("4.33.0"): + from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available + if is_flash_attn_2_available(): + from flash_attn import ( + flash_attn_func, + flash_attn_varlen_func) # pylint: disable=E1101 + from flash_attn.bert_padding import ( + index_first_axis, + pad_input, + unpad_input) # pylint: disable=E1101 + class H2OGPTNeoXAttention(nn.Module): def __init__( self, - model, config, - heavy_ratio, - recent_ratio, - h2o_min_seqlen=1024, - real_drop=False, - is_gen=False, - mean=False, - local=True + h2o_config: H2OConfig, ): - super().__init__() self.config = config self.num_attention_heads = config.num_attention_heads self.hidden_size = config.hidden_size if self.hidden_size % self.num_attention_heads != 0: raise ValueError( - "The hidden size is not divisible by the number of attention heads! Make sure to update them" + "The hidden size is not divisble by the number of attention heads! Make sure to update them" ) self.head_size = self.hidden_size // self.num_attention_heads self.rotary_ndims = int(self.head_size * config.rotary_pct) - self.bias = model.bias + self._init_bias(config.max_position_embeddings) self.register_buffer("masked_bias", torch.tensor(-1e9), persistent=False) - self.rotary_emb = model.rotary_emb + self._init_rope() self.norm_factor = self.head_size**-0.5 - self.query_key_value = model.query_key_value - self.dense = model.dense - self.attention_dropout = model.attention_dropout + self.query_key_value = nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=config.attention_bias) + self.dense = nn.Linear(config.hidden_size, config.hidden_size, bias=config.attention_bias) + self.attention_dropout = nn.Dropout(config.attention_dropout) self.is_causal = True # for h2o if real_drop: real_drop = False logger.warning_once("GPTNeoXAttention not support for kv cache, usning simulation mode.") - self.real_drop = real_drop - self.is_gen = is_gen - self.mean = mean - self.local = local + self.h2o_config = h2o_config + self.is_gen = False + self.mean = h2o_config.mean + self.local = h2o_config.local - self.heavy_ratio = heavy_ratio - self.recent_ratio = recent_ratio - self.h2o_min_seqlen = h2o_min_seqlen + self.heavy_ratio = h2o_config.heavy_ratio + self.recent_ratio = h2o_config.recent_ratio + self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) def forward( self, @@ -200,7 +221,7 @@ def _attn(self, query, key, value, attention_mask=None, head_mask=None): attn_weights = nn.functional.softmax(attn_scores, dim=-1) attn_weights = attn_weights.to(value.dtype) - # get hh mask + # h2o from ..h2o import get_hh_mask mask = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights.detach().clone(), local=self.local) attn_weights[~mask] = torch.finfo(attn_weights.dtype).min @@ -214,6 +235,232 @@ def _attn(self, query, key, value, attention_mask=None, head_mask=None): attn_output = torch.matmul(attn_weights, value) return attn_output, attn_weights +class H2OGPTNeoXFlashAttention2(H2OGPTNeoXAttention): + """ + GPTNeoX flash attention module. This module inherits from `GPTNeoXAttention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: torch.FloatTensor, + position_ids: torch.LongTensor, + head_mask: Optional[torch.FloatTensor] = None, + layer_past: Optional[Tuple[torch.Tensor]] = None, + use_cache: Optional[bool] = False, + output_attentions: Optional[bool] = False, + ): + has_layer_past = layer_past is not None + + # Compute QKV + # Attention heads [batch, seq_len, hidden_size] + # --> [batch, seq_len, (np * 3 * head_size)] + qkv = self.query_key_value(hidden_states) + + # [batch, seq_len, (num_heads * 3 * head_size)] + # --> [batch, seq_len, num_heads, 3 * head_size] + new_qkv_shape = qkv.size()[:-1] + (self.num_attention_heads, 3 * self.head_size) + qkv = qkv.view(*new_qkv_shape) + + # [batch, seq_len, num_attention_heads, 3 * head_size] --> 3 [batch, num_attention_heads, seq_len, head_size] + query = qkv[..., : self.head_size].permute(0, 2, 1, 3) + key = qkv[..., self.head_size : 2 * self.head_size].permute(0, 2, 1, 3) + value = qkv[..., 2 * self.head_size :].permute(0, 2, 1, 3) + + query_length = query.shape[-2] + + # Compute rotary embeddings on rotary_ndims + query_rot = query[..., : self.rotary_ndims] + query_pass = query[..., self.rotary_ndims :] + key_rot = key[..., : self.rotary_ndims] + key_pass = key[..., self.rotary_ndims :] + + # Compute token offset for rotary embeddings (when decoding) + seq_len = key.shape[-2] + if has_layer_past: + seq_len += layer_past[0].shape[-2] + cos, sin = self.rotary_emb(value, seq_len=seq_len) + query, key = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids) + query = torch.cat((query, query_pass), dim=-1) + key = torch.cat((key, key_pass), dim=-1) + + # Cache QKV values + if has_layer_past: + past_key = layer_past[0] + past_value = layer_past[1] + key = torch.cat((past_key, key), dim=-2) + value = torch.cat((past_value, value), dim=-2) + present = (key, value) if use_cache else None + + # GPT-neo-X casts query and key in fp32 to apply rotary embedding in full precision + target_dtype = value.dtype + if query.dtype != target_dtype: + query = query.to(target_dtype) + if key.dtype != target_dtype: + key = key.to(target_dtype) + + # Permute to get the expected shape for Flash Attention + query = query.permute(0, 2, 1, 3) + key = key.permute(0, 2, 1, 3) + value = value.permute(0, 2, 1, 3) + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 / bfloat16 just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + input_dtype = query.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.query_key_value.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query = query.to(target_dtype) + key = key.to(target_dtype) + value = value.to(target_dtype) + + attention_dropout = self.config.attention_dropout if self.training else 0.0 + + # Compute attention + attn_weights = self._flash_attention_forward( + query, key, value, attention_mask, query_length, dropout=attention_dropout, softmax_scale=self.norm_factor + ) + + # h2o + from ..h2o import get_hh_mask + mask = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights.detach().clone(), local=self.local) + attn_weights[~mask] = torch.finfo(attn_weights.dtype).min + + # Reshape outputs + attn_output = attn_weights.reshape( + attn_weights.shape[0], attn_weights.shape[1], self.num_attention_heads * self.head_size + ) + attn_output = self.dense(attn_output) + + outputs = (attn_output, present) + if output_attentions: + outputs += (attn_weights,) + + return outputs + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._flash_attention_forward + def _flash_attention_forward( + self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + """ + if not self._flash_attn_uses_top_left_mask: + causal = self.is_causal + else: + # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. + causal = self.is_causal and query_length != 1 + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + attn_output = flash_attn_func( + query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal + ) + + return attn_output + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._upad_input with num_heads->num_attention_heads + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape + + key_layer = index_first_axis( + key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + value_layer = index_first_axis( + value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k + ) + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, self.num_attention_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): """Applies Rotary Position Embedding to the query and key tensors. @@ -246,3 +493,189 @@ def rotate_half(x): x1 = x[..., : x.shape[-1] // 2] x2 = x[..., x.shape[-1] // 2 :] return torch.cat((-x2, x1), dim=-1) + + +@add_start_docstrings( + """GPTNeoX Model with a `language modeling` head on top for CLM fine-tuning.""", GPT_NEOX_START_DOCSTRING +) +class H2OGPTNeoXForCausalLM(GPTNeoXPreTrainedModel): + _tied_weights_keys = ["embed_out.weight"] + + def __init__(self, config, h2o_config): + super().__init__(config) + + self.gpt_neox = GPTNeoXModel(config) + self.embed_out = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + num_layers = len(self.model.layers) + for layer_idx in range(num_layers): + module = self.gpt_neox.layers[layer_idx].self_attn + cls_name = module.__class__.__name__ + if cls_name == "GPTNeoXFlashAttention2": + cls = H2OGPTNeoXFlashAttention2 + else: + cls = H2OGPTNeoXAttention + + self.gpt_neox.layers[layer_idx].self_attn = cls( + config, + layer_idx, + h2o_config + ) + + # Initialize weights and apply final processing + self.post_init() + + def get_output_embeddings(self): + return self.embed_out + + def set_output_embeddings(self, new_embeddings): + self.embed_out = new_embeddings + + @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional tensors are + only required when the model is used as a decoder in a Sequence to Sequence model. + + Contains pre-computed hidden-states (key and values in the self-attention blocks that can be used (see + `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in + `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are + ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, GPTNeoXForCausalLM, GPTNeoXConfig + >>> import torch + + >>> tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b") + >>> config = GPTNeoXConfig.from_pretrained("EleutherAI/gpt-neox-20b") + >>> config.is_decoder = True + >>> model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/gpt-neox-20b", config=config) + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + + >>> prediction_logits = outputs.logits + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.gpt_neox( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + lm_logits = self.embed_out(hidden_states) + + lm_loss = None + if labels is not None: + # move labels to correct device to enable model parallelism + labels = labels.to(lm_logits.device) + # we are doing next-token prediction; shift prediction scores and input ids by one + shift_logits = lm_logits[:, :-1, :].contiguous() + labels = labels[:, 1:].contiguous() + loss_fct = CrossEntropyLoss() + lm_loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), labels.view(-1)) + + if not return_dict: + output = (lm_logits,) + outputs[1:] + return ((lm_loss,) + output) if lm_loss is not None else output + + return CausalLMOutputWithPast( + loss=lm_loss, + logits=lm_logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + input_shape = input_ids.shape + # cut decoder_input_ids if past is used + if past_key_values is not None: + past_length = past_key_values[0][0].shape[2] + + # Some generation methods already pass only the last input ID + if input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = input_ids.shape[1] - 1 + + input_ids = input_ids[:, remove_prefix_length:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_shape) + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + model_inputs.update( + { + "attention_mask": attention_mask, + "past_key_values": past_key_values, + "position_ids": position_ids, + "use_cache": kwargs.get("use_cache"), + } + ) + + return model_inputs + + def _reorder_cache(self, past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past[:2]) + + layer_past[2:], + ) + return reordered_past \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index d9aaa7a5336..7cc47e31737 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -18,20 +18,43 @@ import math from typing import List, Optional, Tuple, Union +from functools import partial import torch import torch.nn as nn import torch.nn.functional as F - -from transformers.cache_utils import Cache -from transformers.utils import logging +from torch.nn import CrossEntropyLoss + +from transformers.cache_utils import Cache, StaticCache +from transformers.modeling_attn_mask_utils import AttentionMaskConverter +from transformers.utils import ( + add_start_docstrings_to_model_forward, + is_flash_attn_2_available, + is_flash_attn_greater_or_equal_2_10, + logging, + replace_return_docstrings, + ) from transformers.models.llama.configuration_llama import LlamaConfig -from transformers.models.llama.modeling_llama import rotate_half, repeat_kv, _get_unpad_data - -from ..h2o import H2OKVCache +from transformers.models.llama.modeling_llama import ( + rotate_half, + repeat_kv, + _get_unpad_data, + LlamaRotaryEmbedding, + LlamaLinearScalingRotaryEmbedding, + LlamaDynamicNTKScalingRotaryEmbedding, + LlamaPreTrainedModel, + LlamaModel, + LLAMA_INPUTS_DOCSTRING +) +from transformers.modeling_outputs import ( + CausalLMOutputWithPast, +) + +from ..h2o import H2OKVCache, H2OConfig, generate logger = logging.get_logger(__name__) +_CONFIG_FOR_DOC = "LlamaConfig" from packaging import version import transformers @@ -78,19 +101,13 @@ class H2OLlamaAttention(nn.Module): def __init__( self, - model, config: LlamaConfig, - heavy_ratio, - recent_ratio, - h2o_min_seqlen=1024, - real_drop=False, - is_gen=False, - mean=False, - local=True + layer_idx: Optional[int] = None, + h2o_config: H2OConfig = None, ): super().__init__() self.config = config - self.layer_idx = model.layer_idx + self.layer_idx = layer_idx if self.layer_idx is None: logger.warning_once( f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " @@ -114,23 +131,52 @@ def __init__( f" and `num_heads`: {self.num_heads})." ) - self.q_proj = model.q_proj - self.k_proj = model.k_proj - self.v_proj = model.v_proj - self.o_proj = model.o_proj - self.rotary_emb = model.rotary_emb + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias) + self.o_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=config.attention_bias) + self._init_rope() # for h2o - self.is_gen = is_gen - self.real_drop = real_drop - self.mean = mean - self.local = local - - self.heavy_ratio = heavy_ratio - self.recent_ratio = recent_ratio - self.h2o_min_seqlen = h2o_min_seqlen - - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_min_seqlen) + self.h2o_config = h2o_config + self.is_gen = False + self.real_drop = h2o_config.real_drop + self.mean = h2o_config.mean + self.local = h2o_config.local + + self.heavy_ratio = h2o_config.heavy_ratio + self.recent_ratio = h2o_config.recent_ratio + self.h2o_min_seqlen = h2o_config.h2o_min_seqlen + + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) + + def _init_rope(self): + if self.config.rope_scaling is None: + self.rotary_emb = LlamaRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + else: + scaling_type = self.config.rope_scaling["type"] + scaling_factor = self.config.rope_scaling["factor"] + if scaling_type == "linear": + self.rotary_emb = LlamaLinearScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + scaling_factor=scaling_factor, + base=self.rope_theta, + ) + elif scaling_type == "dynamic": + self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + scaling_factor=scaling_factor, + base=self.rope_theta, + ) + else: + raise ValueError(f"Unknown RoPE scaling type {scaling_type}") + def forward( self, @@ -210,6 +256,7 @@ def forward( ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states + self.h2o_kv_cache.past_length += attn_weights.size(-2) else: from ..h2o import get_hh_mask mask = get_hh_mask( @@ -353,6 +400,7 @@ def forward( ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states + self.h2o_kv_cache.past_length += attn_weights.size(-2) attn_output = self._flash_attention_forward( query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate @@ -557,8 +605,12 @@ def forward( past_key_value.value_cache[self.layer_idx], mean=self.mean ) + # if self.layer_idx == 0: + # print(self.layer_idx, self.is_gen, query_states.shape, new_key_states.shape) + # print(query_states.shape, key_states.shape, value_states.shape) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states + self.h2o_kv_cache.past_length += attn_weights.size(-2) # In case we are not compiling, we may set `causal_mask` to None, # which is required to dispatch to SDPA's Flash Attention 2 backend, rather @@ -576,5 +628,319 @@ def forward( attn_output = attn_output.view(bsz, q_len, self.hidden_size) attn_output = self.o_proj(attn_output) + # if self.layer_idx == 0: + # print(attn_output.shape) return attn_output, None, past_key_value + +class H2OLlamaForCausalLM(LlamaPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + def __init__( + self, + config: LlamaConfig, + h2o_config: H2OConfig, + ): + super().__init__(config) + self.model = LlamaModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + num_layers = len(self.model.layers) + for layer_idx in range(num_layers): + module = self.model.layers[layer_idx].self_attn + cls_name = module.__class__.__name__ + if not h2o_config.real_drop: + cls = H2OLlamaAttention + elif cls_name == "LlamaFlashAttention2": + cls = H2OLlamaFlashAttention2 + elif cls_name == "LlamaSdpaAttention": + cls = H2OLlamaSdpaAttention + else: + cls = H2OLlamaAttention + + self.model.layers[layer_idx].self_attn = cls( + config, + layer_idx, + h2o_config + ) + + # Initialize weights and apply final processing + self.post_init() + + self.ori_generate = self.generate + self.generate = partial(generate, self) + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, LlamaForCausalLM + + >>> model = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + ) + + hidden_states = outputs[0] + if self.config.pretraining_tp > 1: + lm_head_slices = self.lm_head.weight.split(self.vocab_size // self.config.pretraining_tp, dim=0) + logits = [F.linear(hidden_states, lm_head_slices[i]) for i in range(self.config.pretraining_tp)] + logits = torch.cat(logits, dim=-1) + else: + logits = self.lm_head(hidden_states) + logits = logits.float() + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + cache_position=None, + use_cache=True, + **kwargs, + ): + past_length = 0 + if past_key_values is not None: + if isinstance(past_key_values, Cache): + past_length = cache_position[0] if cache_position is not None else past_key_values.get_seq_length() + max_cache_length = ( + torch.tensor(past_key_values.get_max_length(), device=input_ids.device) + if past_key_values.get_max_length() is not None + else None + ) + cache_length = past_length if max_cache_length is None else torch.min(max_cache_length, past_length) + # TODO joao: remove this `else` after `generate` prioritizes `Cache` objects + else: + cache_length = past_length = past_key_values[0][0].shape[2] + max_cache_length = None + + # cache_length = past_length = input_ids.shape[-1] - 1 + cache_length = past_length = self.model.layers[0].self_attn.h2o_kv_cache.past_length + max_cache_length = None + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as input) + if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: + input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] + # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard + # input_ids based on the past_length. + elif past_length < input_ids.shape[1]: + input_ids = input_ids[:, past_length:] + # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. + + # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. + if ( + max_cache_length is not None + and attention_mask is not None + and cache_length + input_ids.shape[1] > max_cache_length + ): + attention_mask = attention_mask[:, -max_cache_length:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + # The `contiguous()` here is necessary to have a static stride during decoding. torchdynamo otherwise + # recompiles graphs as the stride of the inputs is a guard. Ref: https://github.com/huggingface/transformers/pull/29114 + # TODO: use `next_tokens` directly instead. + model_inputs = {"input_ids": input_ids.contiguous()} + + input_length = position_ids.shape[-1] if position_ids is not None else input_ids.shape[-1] + if cache_position is None: + cache_position = torch.arange(past_length, past_length + input_length, device=input_ids.device) + elif use_cache: + cache_position = cache_position[-input_length:] + + model_inputs.update( + { + "position_ids": position_ids, + "cache_position": cache_position, + "past_key_values": past_key_values, + "use_cache": use_cache, + "attention_mask": attention_mask, + } + ) + return model_inputs + + def _update_causal_mask( + self, + attention_mask: torch.Tensor, + input_tensor: torch.Tensor, + cache_position: torch.Tensor, + past_key_values: Cache, + output_attentions: bool, + ): + # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static + # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes. + # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using + # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114 + + if self.config._attn_implementation == "flash_attention_2": + if attention_mask is not None and 0.0 in attention_mask: + return attention_mask + return None + + # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in + # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail + # to infer the attention mask. + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + using_static_cache = isinstance(past_key_values, StaticCache) + + # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward + if self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions: + if AttentionMaskConverter._ignore_causal_mask_sdpa( + attention_mask, + inputs_embeds=input_tensor, + past_key_values_length=past_seen_tokens, + is_training=self.training, + ): + return None + + dtype, device = input_tensor.dtype, input_tensor.device + min_dtype = torch.finfo(dtype).min + sequence_length = input_tensor.shape[1] + if using_static_cache: + target_length = past_key_values.get_max_length() + else: + target_length = ( + attention_mask.shape[-1] + if isinstance(attention_mask, torch.Tensor) + else past_seen_tokens + sequence_length + 1 + ) + + if attention_mask is not None and attention_mask.dim() == 4: + # in this case we assume that the mask comes already in inverted form and requires no inversion or slicing + if attention_mask.max() != 0: + raise ValueError("Custom 4D attention mask should be passed in inverted form with max==0`") + causal_mask = attention_mask + else: + causal_mask = torch.full( + (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device + ) + if sequence_length != 1: + causal_mask = torch.triu(causal_mask, diagonal=1) + causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) + causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1) + if attention_mask is not None: + causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit + mask_length = attention_mask.shape[-1] + padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :] + padding_mask = padding_mask == 0 + causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( + padding_mask, min_dtype + ) + if ( + self.config._attn_implementation == "sdpa" + and attention_mask is not None + and attention_mask.device.type == "cuda" + and not output_attentions + ): + # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when + # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. + # Details: https://github.com/pytorch/pytorch/issues/110213 + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + + return causal_mask \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index 6a5e9a4a9e1..468db0d64a1 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -15,19 +15,52 @@ # See the License for the specific language governing permissions and # limitations under the License. import math +import inspect import logging from typing import List, Optional, Tuple, Union +from functools import partial import torch import torch.nn as nn +from torch.nn import CrossEntropyLoss from transformers.cache_utils import Cache -from transformers.models.mistral.modeling_mistral import apply_rotary_pos_emb, repeat_kv - -from ..h2o import H2OKVCache +from transformers.models.mistral.modeling_mistral import ( + apply_rotary_pos_emb, + repeat_kv, + _get_unpad_data, + MistralConfig, + MistralModel, + MistralPreTrainedModel, + MistralRotaryEmbedding, + CausalLMOutputWithPast, + MISTRAL_INPUTS_DOCSTRING, + _CONFIG_FOR_DOC, + + ) +from transformers.file_utils import ( + replace_return_docstrings, + add_start_docstrings_to_model_forward, + ) + +from ..h2o import H2OKVCache, H2OConfig, generate logger = logging.getLogger(__name__) +from packaging import version +import transformers +if version.parse(transformers.__version__) > version.parse("4.33.0"): + from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available + if is_flash_attn_2_available(): + from flash_attn import ( + flash_attn_func, + flash_attn_varlen_func) # pylint: disable=E1101 + from flash_attn.bert_padding import ( + index_first_axis, + pad_input, + unpad_input) # pylint: disable=E1101 + _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) + class H2OMistralAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper. @@ -36,21 +69,14 @@ class H2OMistralAttention(nn.Module): """ def __init__( - self, - model, - config, - heavy_ratio, - recent_ratio, - h2o_min_seqlen=1024, - real_drop=False, - is_gen=False, - mean=False, - local=True + self, config: MistralConfig, + layer_idx: Optional[int] = None, + h2o_config: H2OConfig = None, ): super().__init__() self.config = config - self.layer_idx = model.layer_idx - if self.layer_idx is None: + self.layer_idx = layer_idx + if layer_idx is None: logger.warning_once( f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " @@ -72,23 +98,29 @@ def __init__( f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" f" and `num_heads`: {self.num_heads})." ) - self.q_proj = model.q_proj - self.k_proj = model.k_proj - self.v_proj = model.v_proj - self.o_proj = model.o_proj - self.rotary_emb = model.rotary_emb + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + + self.rotary_emb = MistralRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) # for h2o - self.is_gen = is_gen - self.real_drop = real_drop - self.mean = mean - self.local = local + self.h2o_config = h2o_config + self.is_gen = False + self.real_drop = h2o_config.real_drop + self.mean = h2o_config.mean + self.local = h2o_config.local - self.heavy_ratio = heavy_ratio - self.recent_ratio = recent_ratio - self.h2o_min_seqlen = h2o_min_seqlen + self.heavy_ratio = h2o_config.heavy_ratio + self.recent_ratio = h2o_config.recent_ratio + self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -166,6 +198,7 @@ def forward( ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states + self.h2o_kv_cache.past_length += attn_weights.size(-2) else: from ..h2o import get_hh_mask mask = get_hh_mask( @@ -197,6 +230,318 @@ def forward( return attn_output, attn_weights, past_key_value +class H2OMistralFlashAttention2(H2OMistralAttention): + """ + Mistral flash attention module. This module inherits from `MistralAttention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + ): + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + + # Because the input can be padded, the absolute sequence length depends on the max position id. + rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1 + cos, sin = self.rotary_emb(value_states, seq_len=rotary_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + use_sliding_windows = ( + _flash_supports_window_size + and getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + ) + + if not _flash_supports_window_size: + logger.warning_once( + "The current flash attention version does not support sliding window attention, for a more memory efficient implementation" + " make sure to upgrade flash-attn library." + ) + + if past_key_value is not None: + # Activate slicing cache only if the config has a value `sliding_windows` attribute + cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 + if ( + getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + and cache_has_contents + ): + slicing_tokens = 1 - self.config.sliding_window + + past_key = past_key_value[self.layer_idx][0] + past_value = past_key_value[self.layer_idx][1] + + past_key = past_key[:, :, slicing_tokens:, :].contiguous() + past_value = past_value[:, :, slicing_tokens:, :].contiguous() + + if past_key.shape[-2] != self.config.sliding_window - 1: + raise ValueError( + f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" + f" {past_key.shape}" + ) + + if attention_mask is not None: + attention_mask = attention_mask[:, slicing_tokens:] + attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) + + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + dropout_rate = 0.0 if not self.training else self.attention_dropout + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 just to be sure everything works as expected. + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + # Reashape to the expected shape for Flash Attention + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + # h2o + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + + if past_key_value is not None: + if not self.is_gen: + self.h2o_kv_cache.clean_scores() + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value.key_cache[self.layer_idx], + past_key_value.value_cache[self.layer_idx], + mean=self.mean + ) + past_key_value.key_cache[self.layer_idx] = new_key_states + past_key_value.value_cache[self.layer_idx] = new_value_states + self.h2o_kv_cache.past_length += attn_weights.size(-2) + + attn_output = self._flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + q_len, + dropout=dropout_rate, + use_sliding_windows=use_sliding_windows, + ) + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + def _flash_attention_forward( + self, + query_states, + key_states, + value_states, + attention_mask, + query_length, + dropout=0.0, + softmax_scale=None, + use_sliding_windows=False, + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + use_sliding_windows (`bool`, *optional*): + Whether to activate sliding window attention. + """ + if not self._flash_attn_uses_top_left_mask: + causal = self.is_causal + else: + # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. + causal = self.is_causal and query_length != 1 + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + if not use_sliding_windows: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + if not use_sliding_windows: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + return attn_output + + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape + + # On the first iteration we need to properly re-create the padding mask + # by slicing it on the proper place + if kv_seq_len != attention_mask.shape[-1]: + attention_mask_num_tokens = attention_mask.shape[-1] + attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :] + + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + + key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + class H2OMistralSdpaAttention(H2OMistralAttention): """ Mistral attention module using torch.nn.functional.scaled_dot_product_attention. @@ -295,6 +640,7 @@ def forward( ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states + self.h2o_kv_cache.past_length += attn_weights.size(-2) if query_states.device.type == "cuda" and attention_mask is not None: query_states = query_states.contiguous() @@ -318,3 +664,216 @@ def forward( attn_output = self.o_proj(attn_output) return attn_output, None, past_key_value + +class MistralForCausalLM(MistralPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + def __init__( + self, + config, + h2o_config: H2OConfig): + super().__init__(config) + self.model = MistralModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + num_layers = len(self.model.layers) + for layer_idx in range(num_layers): + module = self.model.layers[layer_idx].self_attn + cls_name = module.__class__.__name__ + if not h2o_config.real_drop: + cls = H2OMistralAttention + elif cls_name == "MistralFlashAttention2": + cls = H2OMistralFlashAttention2 + elif cls_name == "LlamaSdpaAttention": + cls = H2OMistralSdpaAttention + else: + cls = H2OMistralAttention + + self.model.layers[layer_idx].self_attn = cls( + config, + layer_idx, + h2o_config + ) + + # Initialize weights and apply final processing + self.post_init() + + self.ori_generate = self.generate + self.generate = partial(generate, self) + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, MistralForCausalLM + + >>> model = MistralForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1") + >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + logits = logits.float() + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Ensure tensors are on the same device + shift_labels = shift_labels.to(shift_logits.device) + loss_fct = CrossEntropyLoss() + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + # Omit tokens covered by past_key_values + if past_key_values is not None: + if isinstance(past_key_values, Cache): + cache_length = past_key_values.get_seq_length() + past_length = past_key_values.seen_tokens + max_cache_length = past_key_values.get_max_length() + else: + cache_length = past_length = past_key_values[0][0].shape[2] + max_cache_length = None + + cache_length = past_length = self.model.layers[0].self_attn.h2o_kv_cache.past_length + max_cache_length = None + + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as + # input) + if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: + input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] + # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard + # input_ids based on the past_length. + elif past_length < input_ids.shape[1]: + input_ids = input_ids[:, past_length:] + # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. + + # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. + if ( + max_cache_length is not None + and attention_mask is not None + and cache_length + input_ids.shape[1] > max_cache_length + ): + attention_mask = attention_mask[:, -max_cache_length:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "position_ids": position_ids, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index 3c0fd4f7e1e..16213841b45 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -18,15 +18,33 @@ import math import logging from typing import List, Optional, Tuple, Union +from functools import partial import torch import torch.nn as nn +from torch.nn import CrossEntropyLoss from transformers.cache_utils import Cache -from transformers.models.mixtral.modeling_mixtral import apply_rotary_pos_emb, repeat_kv, _get_unpad_data +from transformers.models.mixtral.modeling_mixtral import ( + apply_rotary_pos_emb, + repeat_kv, + _get_unpad_data, + load_balancing_loss_func, + MixtralConfig, + MixtralRotaryEmbedding, + MixtralModel, + MixtralPreTrainedModel, + MoeCausalLMOutputWithPast, + MIXTRAL_INPUTS_DOCSTRING, + _CONFIG_FOR_DOC, + ) +from transformers.file_utils import ( + replace_return_docstrings, + add_start_docstrings_to_model_forward, + ) from transformers.utils import is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10 -from ..h2o import H2OKVCache +from ..h2o import H2OKVCache, H2OConfig, generate logger = logging.getLogger(__name__) @@ -44,21 +62,14 @@ class H2OMixtralAttention(nn.Module): """ def __init__( - self, - model, - config, - heavy_ratio, - recent_ratio, - h2o_min_seqlen=1024, - real_drop=False, - is_gen=False, - mean=False, - local=True + self, config: MixtralConfig, + layer_idx: Optional[int] = None, + h2o_config: H2OConfig = None, ): super().__init__() self.config = config - self.layer_idx = model.layer_idx - if self.layer_idx is None: + self.layer_idx = layer_idx + if layer_idx is None: logger.warning_once( f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " @@ -80,24 +91,29 @@ def __init__( f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" f" and `num_heads`: {self.num_heads})." ) - self.q_proj = model.q_proj - self.k_proj = model.k_proj - self.v_proj = model.v_proj - self.o_proj = model.o_proj - - self.rotary_emb = model.rotary_emb + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + + self.rotary_emb = MixtralRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) # for h2o - self.is_gen = is_gen - self.real_drop = real_drop - self.mean = mean - self.local = local + self.h2o_config = h2o_config + self.is_gen = False + self.real_drop = h2o_config.real_drop + self.mean = h2o_config.mean + self.local = h2o_config.local - self.heavy_ratio = heavy_ratio - self.recent_ratio = recent_ratio - self.h2o_min_seqlen = h2o_min_seqlen + self.heavy_ratio = h2o_config.heavy_ratio + self.recent_ratio = h2o_config.recent_ratio + self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -176,6 +192,7 @@ def forward( ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states + self.h2o_kv_cache.past_length += attn_weights.size(-2) else: from ..h2o import get_hh_mask mask = get_hh_mask( @@ -330,6 +347,7 @@ def forward( ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states + self.h2o_kv_cache.past_length += attn_weights.size(-2) # In PEFT, usually we cast the layer norms in float32 for training stability reasons # therefore the input hidden states gets silently casted in float32. Hence, we need @@ -601,6 +619,7 @@ def forward( ) past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states + self.h2o_kv_cache.past_length += attn_weights.size(-2) if attention_mask is not None: if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): @@ -630,4 +649,245 @@ def forward( attn_output = self.o_proj(attn_output) - return attn_output, None, past_key_value \ No newline at end of file + return attn_output, None, past_key_value + + +class MixtralForCausalLM(MixtralPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config, h2o_config: H2OConfig): + super().__init__(config) + self.model = MixtralModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.router_aux_loss_coef = config.router_aux_loss_coef + self.num_experts = config.num_local_experts + self.num_experts_per_tok = config.num_experts_per_tok + + num_layers = len(self.model.layers) + for layer_idx in range(num_layers): + module = self.model.layers[layer_idx].self_attn + cls_name = module.__class__.__name__ + if not h2o_config.real_drop: + cls = H2OMixtralAttention + elif cls_name == "MixtralFlashAttention2": + cls = H2OMixtralFlashAttention2 + elif cls_name == "MixtralSdpaAttention": + cls = H2OMixtralSdpaAttention + else: + cls = H2OMixtralAttention + + self.model.layers[layer_idx].self_attn = cls( + config, + layer_idx, + h2o_config + ) + + # Initialize weights and apply final processing + self.post_init() + + self.ori_generate = self.generate + self.generate = partial(generate, self) + + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + # Ignore copy + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, MoeCausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, MixtralForCausalLM + + >>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1") + >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_router_logits = ( + output_router_logits if output_router_logits is not None else self.config.output_router_logits + ) + + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + output_router_logits=output_router_logits, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + logits = logits.float() + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + aux_loss = None + if output_router_logits: + aux_loss = load_balancing_loss_func( + outputs.router_logits if return_dict else outputs[-1], + self.num_experts, + self.num_experts_per_tok, + attention_mask, + ) + if labels is not None: + loss += self.router_aux_loss_coef * aux_loss.to(loss.device) # make sure to reside in the same device + + if not return_dict: + output = (logits,) + outputs[1:] + if output_router_logits: + output = (aux_loss,) + output + return (loss,) + output if loss is not None else output + + return MoeCausalLMOutputWithPast( + loss=loss, + aux_loss=aux_loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + router_logits=outputs.router_logits, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + output_router_logits=False, + **kwargs, + ): + # Omit tokens covered by past_key_values + if past_key_values is not None: + if isinstance(past_key_values, Cache): + cache_length = past_key_values.get_seq_length() + past_length = past_key_values.seen_tokens + max_cache_length = past_key_values.get_max_length() + else: + cache_length = past_length = past_key_values[0][0].shape[2] + max_cache_length = None + + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as + # input) + if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: + input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] + # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard + # input_ids based on the past_length. + elif past_length < input_ids.shape[1]: + input_ids = input_ids[:, past_length:] + # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. + + # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. + if ( + max_cache_length is not None + and attention_mask is not None + and cache_length + input_ids.shape[1] > max_cache_length + ): + attention_mask = attention_mask[:, -max_cache_length:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "position_ids": position_ids, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + "output_router_logits": output_router_logits, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index ff5ef8ace53..0780c3414a5 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -16,29 +16,34 @@ # limitations under the License. """PyTorch OPT model.""" from typing import List, Optional, Tuple, Union +from functools import partial import torch import torch.nn as nn +from torch.nn import CrossEntropyLoss, MSELoss from ..h2o import get_hh_mask, H2OKVCache -from transformers.utils import is_flash_attn_greater_or_equal_2_10, logging +from transformers.utils import ( + is_flash_attn_greater_or_equal_2_10, + logging, + replace_return_docstrings + ) +from transformers.modeling_outputs import CausalLMOutputWithPast +from transformers.models.opt.modeling_opt import OPTPreTrainedModel, OPTModel, OPTConfig + +from ..h2o import H2OKVCache, H2OConfig, generate logger = logging.get_logger(__name__) +_CONFIG_FOR_DOC = "OPTConfig" class H2OOPTAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__( self, - model, - config, - heavy_ratio, - recent_ratio, - h2o_min_seqlen=1024, - real_drop=False, - is_gen=False, - mean=False, - local=True + config: OPTConfig, + is_decoder: bool = False, + h2o_config: H2OConfig =- None, ): super().__init__() self.config = config @@ -56,24 +61,25 @@ def __init__( f" and `num_heads`: {self.num_heads})." ) self.scaling = self.head_dim**-0.5 - self.is_decoder = model.is_decoder + self.is_decoder = is_decoder - self.k_proj = model.k_proj - self.v_proj = model.v_proj - self.q_proj = model.q_proj - self.out_proj = model.out_proj + self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=self.enable_bias) + self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=self.enable_bias) + self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=self.enable_bias) + self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=self.enable_bias) # for h2o - self.is_gen = is_gen - self.real_drop = real_drop - self.mean = mean - self.local = local + self.h2o_config = h2o_config + self.is_gen = False + self.real_drop = h2o_config.real_drop + self.mean = h2o_config.mean + self.local = h2o_config.local - self.heavy_ratio = heavy_ratio - self.recent_ratio = recent_ratio - self.h2o_min_seqlen = h2o_min_seqlen + self.heavy_ratio = h2o_config.heavy_ratio + self.recent_ratio = h2o_config.recent_ratio + self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, real_drop, h2o_min_seqlen) + self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -163,8 +169,8 @@ def forward( mean=self.mean ) past_key_value = (new_key_states, new_value_states) + self.h2o_kv_cache.past_length += attn_weights.size(-2) else: - from ..h2o import get_hh_mask mask = get_hh_mask( self.heavy_ratio, self.recent_ratio, @@ -367,23 +373,14 @@ def forward( if not self.is_gen: self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - mean=self.mean - ) - past_key_value = (new_key_states, new_value_states) - else: - mask = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - mean=self.mean - ) - key_states = key_states * mask.unsqueeze(-1) - value_states = value_states * mask.unsqueeze(-1) + new_key_states, new_value_states = self.h2o_kv_cache( + attn_weights, + past_key_value[0], + past_key_value[1], + mean=self.mean + ) + past_key_value = (new_key_states, new_value_states) + self.h2o_kv_cache.past_length += attn_weights.size(-2) attn_output = self._flash_attention_forward( query_states, key_states, value_states, attention_mask, query_length, dropout=attn_dropout @@ -396,3 +393,229 @@ def forward( attn_weights_reshaped = None return attn_output, attn_weights_reshaped, past_key_value + + +class H2OOPTForCausalLM(OPTPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + def __init__( + self, + config, + h2o_config: H2OConfig, + ): + super().__init__(config) + self.model = OPTModel(config) + + # the lm_head weight is automatically tied to the embed tokens weight + self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False) + + num_layers = len(self.model.decoder.layers) + for layer_idx in range(num_layers): + module = self.model.layers[layer_idx].self_attn + cls_name = module.__class__.__name__ + if not h2o_config.real_drop: + cls = H2OOPTAttention + elif cls_name == "OptFlashAttention2": + cls = H2OOptFlashAttention2 + else: + cls = H2OOPTAttention + self.model.decoder.layers[layer_idx].self_attn = cls( + config, + is_decoder=True, + h2o_config=h2o_config + ) + # Initialize weights and apply final processing + self.post_init() + + self.ori_generate = self.generate + self.generate = partial(generate, self) + + def get_input_embeddings(self): + return self.model.decoder.embed_tokens + + def set_input_embeddings(self, value): + self.model.decoder.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model.decoder = decoder + + def get_decoder(self): + return self.model.decoder + + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional + tensors are only required when the model is used as a decoder in a Sequence to Sequence model. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, OPTForCausalLM + + >>> model = OPTForCausalLM.from_pretrained("facebook/opt-350m") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious. I'm just a little bit of a weirdo." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model.decoder( + input_ids=input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + logits = self.lm_head(outputs[0]).contiguous() + + loss = None + if labels is not None: + # move labels to correct device to enable model parallelism + labels = labels.to(logits.device) + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + if past_key_values is not None: + # past_length = past_key_values[0][0].shape[2] + past_length = self.model.decoder.layers[0].self_attn.h2o_kv_cache.past_length + + # Some generation methods already pass only the last input ID + if input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = input_ids.shape[1] - 1 + + input_ids = input_ids[:, remove_prefix_length:] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past \ No newline at end of file From 5181afc57f0fd03bd11149adbaabe5289f453634 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 18 Jun 2024 02:08:38 -0400 Subject: [PATCH 43/62] fix Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/models/modeling_mistral.py | 2 +- .../modeling/kv_cache_compression/models/modeling_mixtral.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index 468db0d64a1..5fff8c241c1 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -665,7 +665,7 @@ def forward( return attn_output, None, past_key_value -class MistralForCausalLM(MistralPreTrainedModel): +class H2OMistralForCausalLM(MistralPreTrainedModel): _tied_weights_keys = ["lm_head.weight"] def __init__( diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index 16213841b45..d54fcc0d9bb 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -652,7 +652,7 @@ def forward( return attn_output, None, past_key_value -class MixtralForCausalLM(MixtralPreTrainedModel): +class H2OMixtralForCausalLM(MixtralPreTrainedModel): _tied_weights_keys = ["lm_head.weight"] def __init__(self, config, h2o_config: H2OConfig): From 58bfcd0d3077dcd6980af4d6082f5f7d804914c8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 06:10:44 +0000 Subject: [PATCH 44/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../modeling/kv_cache_compression/h2o.py | 8 ++++---- .../models/modeling_bloom.py | 17 ++++++++-------- .../models/modeling_gpt_neox.py | 19 +++++++++--------- .../models/modeling_llama.py | 20 +++++++++---------- .../models/modeling_mistral.py | 11 +++++----- .../models/modeling_mixtral.py | 2 +- .../models/modeling_opt.py | 4 ++-- 7 files changed, 41 insertions(+), 40 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py index cea86201cd6..a83b83f5e9d 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py @@ -167,7 +167,7 @@ def __init__( self.hh_score = None self.min_seqlen = min_seqlen self.idx = 0 - + self._past_length = 0 @@ -234,15 +234,15 @@ def clean_scores(self): self.idx = 0 self.past_length = 0 self.hh_score = None - + @property def past_length(self): return self._past_length - + @past_length.setter def past_length(self, value): self._past_length = value - + class H2OConfig(dict): def __init__( self, diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py index 2494208d2d2..07cf8500096 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py @@ -287,7 +287,7 @@ def prepare_inputs_for_generation( input_ids = input_ids[:, remove_prefix_length:] - # the cache may be in the stardard format (e.g. in contrastive search), convert to bloom's format if needed + # the cache may be in the standard format (e.g. in contrastive search), convert to bloom's format if needed if past_key_values[0][0].shape[0] == input_ids.shape[0]: past_key_values = self._convert_to_bloom_cache(past_key_values) @@ -326,11 +326,11 @@ def forward( return_dict: Optional[bool] = None, **deprecated_arguments, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: - r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set - `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` - are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + r"""Labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + + Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set + `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` + are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` """ if deprecated_arguments.pop("position_ids", False) is not False: # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` @@ -388,8 +388,7 @@ def forward( def _reorder_cache( self, past: Tuple[Tuple[torch.Tensor, torch.Tensor], ...], beam_idx: torch.LongTensor ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]: - """ - This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or + """This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct beam_idx at every generation step. @@ -408,4 +407,4 @@ def _reorder_cache( ) for layer_past in standardized_past ) - return self._convert_to_bloom_cache(reordered_past) \ No newline at end of file + return self._convert_to_bloom_cache(reordered_past) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py index 3aa13dbca05..9e5fa4eb6ca 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py @@ -64,7 +64,7 @@ def __init__( self.hidden_size = config.hidden_size if self.hidden_size % self.num_attention_heads != 0: raise ValueError( - "The hidden size is not divisble by the number of attention heads! Make sure to update them" + "The hidden size is not divisible by the number of attention heads! Make sure to update them" ) self.head_size = self.hidden_size // self.num_attention_heads self.rotary_ndims = int(self.head_size * config.rotary_pct) @@ -236,8 +236,9 @@ def _attn(self, query, key, value, attention_mask=None, head_mask=None): return attn_output, attn_weights class H2OGPTNeoXFlashAttention2(H2OGPTNeoXAttention): - """ - GPTNeoX flash attention module. This module inherits from `GPTNeoXAttention` as the weights of the module stays + """GPTNeoX flash attention module. + + This module inherits from `GPTNeoXAttention` as the weights of the module stays untouched. The only required change would be on the forward pass where it needs to correctly call the public API of flash attention and deal with padding tokens in case the input contains any of them. """ @@ -246,7 +247,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() @@ -515,7 +516,7 @@ def __init__(self, config, h2o_config): cls = H2OGPTNeoXFlashAttention2 else: cls = H2OGPTNeoXAttention - + self.gpt_neox.layers[layer_idx].self_attn = cls( config, layer_idx, @@ -547,8 +548,7 @@ def forward( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithPast]: - r""" - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + r"""past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional tensors are @@ -585,7 +585,8 @@ def forward( >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits - ```""" + ``` + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.gpt_neox( @@ -678,4 +679,4 @@ def _reorder_cache(self, past_key_values, beam_idx): tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past[:2]) + layer_past[2:], ) - return reordered_past \ No newline at end of file + return reordered_past diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index 7cc47e31737..e3374db3802 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -149,7 +149,7 @@ def __init__( self.h2o_min_seqlen = h2o_config.h2o_min_seqlen self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) - + def _init_rope(self): if self.config.rope_scaling is None: self.rotary_emb = LlamaRotaryEmbedding( @@ -176,7 +176,7 @@ def _init_rope(self): ) else: raise ValueError(f"Unknown RoPE scaling type {scaling_type}") - + def forward( self, @@ -645,7 +645,7 @@ def __init__( self.model = LlamaModel(config) self.vocab_size = config.vocab_size self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - + num_layers = len(self.model.layers) for layer_idx in range(num_layers): module = self.model.layers[layer_idx].self_attn @@ -658,19 +658,19 @@ def __init__( cls = H2OLlamaSdpaAttention else: cls = H2OLlamaAttention - + self.model.layers[layer_idx].self_attn = cls( config, layer_idx, h2o_config ) - + # Initialize weights and apply final processing self.post_init() self.ori_generate = self.generate self.generate = partial(generate, self) - + def get_input_embeddings(self): return self.model.embed_tokens @@ -688,7 +688,7 @@ def set_decoder(self, decoder): def get_decoder(self): return self.model - + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) def forward( @@ -783,7 +783,7 @@ def forward( hidden_states=outputs.hidden_states, attentions=outputs.attentions, ) - + def prepare_inputs_for_generation( self, input_ids, @@ -864,7 +864,7 @@ def prepare_inputs_for_generation( } ) return model_inputs - + def _update_causal_mask( self, attention_mask: torch.Tensor, @@ -943,4 +943,4 @@ def _update_causal_mask( # Details: https://github.com/pytorch/pytorch/issues/110213 causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) - return causal_mask \ No newline at end of file + return causal_mask diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py index 44e48ada852..2451f94d190 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py @@ -231,8 +231,9 @@ def forward( return attn_output, attn_weights, past_key_value class H2OMistralFlashAttention2(H2OMistralAttention): - """ - Mistral flash attention module. This module inherits from `MistralAttention` as the weights of the module stays + """Mistral flash attention module. + + This module inherits from `MistralAttention` as the weights of the module stays untouched. The only required change would be on the forward pass where it needs to correctly call the public API of flash attention and deal with padding tokens in case the input contains any of them. """ @@ -242,7 +243,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() @@ -689,7 +690,7 @@ def __init__( cls = H2OMistralSdpaAttention else: cls = H2OMistralAttention - + self.model.layers[layer_idx].self_attn = cls( config, layer_idx, @@ -876,4 +877,4 @@ def _reorder_cache(past_key_values, beam_idx): reordered_past += ( tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), ) - return reordered_past \ No newline at end of file + return reordered_past diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py index 3a92f654071..168db78b860 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py @@ -678,7 +678,7 @@ def __init__(self, config, h2o_config: H2OConfig): cls = H2OMixtralSdpaAttention else: cls = H2OMixtralAttention - + self.model.layers[layer_idx].self_attn = cls( config, layer_idx, diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py index 6f2b3d78465..7473a3f091d 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py @@ -43,7 +43,7 @@ def __init__( self, config: OPTConfig, is_decoder: bool = False, - h2o_config: H2OConfig =- None, + h2o_config: H2OConfig =- None, ): super().__init__() self.config = config @@ -618,4 +618,4 @@ def _reorder_cache(past_key_values, beam_idx): reordered_past += ( tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), ) - return reordered_past \ No newline at end of file + return reordered_past From 91c5f3caafb1882915997a9b1c32994cf44b216c Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Thu, 20 Jun 2024 03:36:53 -0400 Subject: [PATCH 45/62] new api Signed-off-by: n1ck-guo --- .../models/modeling_llama.py | 186 ++++++++++-------- .../kv_cache_compression/prune/__init__.py | 2 + .../kv_cache_compression/prune/base.py | 30 +++ .../kv_cache_compression/{ => prune}/h2o.py | 180 ++++++++--------- 4 files changed, 220 insertions(+), 178 deletions(-) create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py rename intel_extension_for_transformers/transformers/modeling/kv_cache_compression/{ => prune}/h2o.py (74%) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index e3374db3802..fd98d194667 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -50,8 +50,8 @@ CausalLMOutputWithPast, ) -from ..h2o import H2OKVCache, H2OConfig, generate - +from intel_extension_for_transformers.transformers.modeling.modeling_gaudi import adapt_transformers_to_gaudi +from ..prune import PruneConfig, H2OConfig logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "LlamaConfig" @@ -96,14 +96,13 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): k_embed = (k * cos) + (rotate_half(k) * sin) return q_embed, k_embed -class H2OLlamaAttention(nn.Module): +class LlamaAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper.""" def __init__( self, config: LlamaConfig, layer_idx: Optional[int] = None, - h2o_config: H2OConfig = None, ): super().__init__() self.config = config @@ -137,18 +136,14 @@ def __init__( self.o_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=config.attention_bias) self._init_rope() - # for h2o - self.h2o_config = h2o_config - self.is_gen = False - self.real_drop = h2o_config.real_drop - self.mean = h2o_config.mean - self.local = h2o_config.local - - self.heavy_ratio = h2o_config.heavy_ratio - self.recent_ratio = h2o_config.recent_ratio - self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) + self._init_func = [] + + def register_init_func(self, func): + self._init_func.append(func) + + def post_init(self): + for func in self._init_func: + func(self) def _init_rope(self): if self.config.rope_scaling is None: @@ -177,7 +172,6 @@ def _init_rope(self): else: raise ValueError(f"Unknown RoPE scaling type {scaling_type}") - def forward( self, hidden_states: torch.Tensor, @@ -239,33 +233,46 @@ def forward( attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + causal_mask = None if attention_mask is not None: # no matter the length, we just slice it causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] attn_weights = attn_weights + causal_mask - # H2O - if past_key_value is not None: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights.detach().clone(), - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean + # pruning kv cache + if self.pruner.real_drop: + if self.layer_idx == 0: + self.pruner.past_length += query_states.size(-2) + if past_key_value is not None: + new_key_states, new_value_states = self.pruner.prune( + self, + query_states, + key_states, + value_states, + causal_mask=causal_mask ) + # reshape kv cache + if self.num_key_value_groups > 1: + n_rep = self.num_key_value_groups + drop_mask = torch.tensor( + [True if i % n_rep == 0 else False for i in range(0, new_key_states.size(1))] + ).repeat(new_key_states.size(0), 1).to(new_key_states.device) + new_key_states = new_key_states[drop_mask].view( + new_key_states.shape[0], + int(new_key_states.shape[1] / n_rep), + -1, + new_key_states.shape[-1]) + new_value_states = new_value_states[drop_mask].view( + new_value_states.shape[0], + int(new_value_states.shape[1] / n_rep), + -1, + new_value_states.shape[-1]) + past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states - self.h2o_kv_cache.past_length += attn_weights.size(-2) - else: - from ..h2o import get_hh_mask - mask = get_hh_mask( - self.heavy_ratio, - self.recent_ratio, - attn_weights.detach().clone(), - local=self.local) - attn_weights[~mask] = torch.finfo(attn_weights.dtype).min - + else: # similuate pruning to calculate acc + mask = self.pruner.get_mask(self, query_states, key_states, value_states, + causal_mask=causal_mask) + attn_weights[~mask] = torch.finfo(attn_weights.dtype).min # upcast attention to fp32 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) @@ -294,7 +301,7 @@ def forward( return attn_output, attn_weights, past_key_value -class H2OLlamaFlashAttention2(H2OLlamaAttention): +class LlamaFlashAttention2(LlamaAttention): """Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays @@ -389,18 +396,20 @@ def forward( # h2o + # pruning kv cache + if self.layer_idx == 0: + self.pruner.past_length += query_states.size(-2) if past_key_value is not None: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean + new_key_states, new_value_states = self.pruner.prune( + self, + query_states, + key_states, + value_states, + causal_mask=causal_mask ) + past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states - self.h2o_kv_cache.past_length += attn_weights.size(-2) attn_output = self._flash_attention_forward( query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate @@ -510,7 +519,7 @@ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query (max_seqlen_in_batch_q, max_seqlen_in_batch_k), ) -class H2OLlamaSdpaAttention(H2OLlamaAttention): +class LlamaSdpaAttention(LlamaAttention): """Llama attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from @@ -589,28 +598,36 @@ def forward( key_states = key_states.contiguous() value_states = value_states.contiguous() - # h2o - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attention_mask is not None: # no matter the length, we just slice it - causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] - attn_weights = attn_weights + causal_mask - + # pruning kv cache + if self.layer_idx == 0: + self.pruner.past_length += query_states.size(-2) if past_key_value is not None: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean + new_key_states, new_value_states = self.pruner.prune( + self, + query_states, + key_states, + value_states, + causal_mask=causal_mask ) - # if self.layer_idx == 0: - # print(self.layer_idx, self.is_gen, query_states.shape, new_key_states.shape) - # print(query_states.shape, key_states.shape, value_states.shape) + # reshape kv cache + if self.num_key_value_groups > 1: + n_rep = self.num_key_value_groups + drop_mask = torch.tensor( + [True if i % n_rep == 0 else False for i in range(0, new_key_states.size(1))] + ).repeat(new_key_states.size(0), 1).to(new_key_states.device) + new_key_states = new_key_states[drop_mask].view( + new_key_states.shape[0], + int(new_key_states.shape[1] / n_rep), + -1, + new_key_states.shape[-1]) + new_value_states = new_value_states[drop_mask].view( + new_value_states.shape[0], + int(new_value_states.shape[1] / n_rep), + -1, + new_value_states.shape[-1]) + past_key_value.key_cache[self.layer_idx] = new_key_states past_key_value.value_cache[self.layer_idx] = new_value_states - self.h2o_kv_cache.past_length += attn_weights.size(-2) # In case we are not compiling, we may set `causal_mask` to None, # which is required to dispatch to SDPA's Flash Attention 2 backend, rather @@ -633,43 +650,58 @@ def forward( return attn_output, None, past_key_value -class H2OLlamaForCausalLM(LlamaPreTrainedModel): +class LlamaForCausalLM(LlamaPreTrainedModel): _tied_weights_keys = ["lm_head.weight"] def __init__( self, config: LlamaConfig, - h2o_config: H2OConfig, + prune_config: PruneConfig, ): super().__init__(config) self.model = LlamaModel(config) self.vocab_size = config.vocab_size self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + if isinstance(prune_config, H2OConfig): + from ..prune import H2OKVPruner + self.pruner = H2OKVPruner(prune_config) + else: + from ..prune import KVPruner + self.pruner = KVPruner(prune_config) + num_layers = len(self.model.layers) for layer_idx in range(num_layers): module = self.model.layers[layer_idx].self_attn cls_name = module.__class__.__name__ - if not h2o_config.real_drop: - cls = H2OLlamaAttention + if not prune_config.real_drop: + cls = LlamaAttention elif cls_name == "LlamaFlashAttention2": - cls = H2OLlamaFlashAttention2 + cls = LlamaFlashAttention2 elif cls_name == "LlamaSdpaAttention": - cls = H2OLlamaSdpaAttention + cls = LlamaSdpaAttention else: - cls = H2OLlamaAttention - + cls = LlamaAttention self.model.layers[layer_idx].self_attn = cls( config, - layer_idx, - h2o_config + layer_idx ) + self.model.layers[layer_idx].self_attn.register_init_func(self.pruner.self_attn_init) + self.model.layers[layer_idx].self_attn.post_init() + self.model.layers[layer_idx].self_attn.pruner = self.pruner + # Initialize weights and apply final processing self.post_init() + def _generate(**kwargs): + self.pruner.before_generate(self, **kwargs) + result = self.ori_generate(**kwargs) + self.pruner.after_generate(self, **kwargs) + return result + self.ori_generate = self.generate - self.generate = partial(generate, self) + self.generate = _generate def get_input_embeddings(self): return self.model.embed_tokens @@ -810,7 +842,7 @@ def prepare_inputs_for_generation( max_cache_length = None # cache_length = past_length = input_ids.shape[-1] - 1 - cache_length = past_length = self.model.layers[0].self_attn.h2o_kv_cache.past_length + cache_length = past_length = self.pruner.past_length max_cache_length = None # Keep only the unprocessed tokens: # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py new file mode 100644 index 00000000000..33a5058953d --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py @@ -0,0 +1,2 @@ +from .base import PruneConfig, KVPruner +from .h2o import H2OConfig, H2OKVPruner \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py new file mode 100644 index 00000000000..e4862c3aa77 --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py @@ -0,0 +1,30 @@ +class PruneConfig(dict): + def __init__(self, real_drop=True): + self.real_drop = real_drop + +class KVPruner: + def __init__(self, prune_config) -> None: + self._past_length = 0 + + def self_attn_init(self, module): + pass + + def prune(self, module, query_states, key_states, value_states, **kwargs): + pass + + def before_generate(self, model, **kwargs): + self.past_length = 0 + + def after_generate(self, model, **kwargs): + pass + + def get_mask(self, model, **kwargs): + pass + + @property + def past_length(self): + return self._past_length + + @past_length.setter + def past_length(self, value): + self._past_length = value \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py similarity index 74% rename from intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py rename to intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py index a83b83f5e9d..e3de4a605da 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py @@ -14,75 +14,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import importlib -from functools import partial - +import math import torch -from torch import nn - -SIM_CLS_MAPPING = { - 'llama': 'H2OLlamaAttention', - 'opt': 'H2OOPTAttention', - 'bloom': 'H2OBloomAttention', - 'mistral': 'H2OMistralAttention', - 'mixtral': 'H2OMixtralAttention', - 'gpt_neox': 'H2OGPTNeoXAttention', -} - -def set_module(model, op_name, new_module): - """Set module with a given op name. - - Args: - model (object): the input model. - op_name (str): name of op. - new_module (object): the input model. - - Returns: - module (object). - """ - module = model - name_list = op_name.split(".") - for name in name_list[:-1]: - if hasattr(module, name): - module = getattr(module, name) - else: - module = module - setattr(module, name_list[-1], new_module) - -def get_module(model, op_name): - """Get module from model by key name. - - Args: - model (torch.nn.Module): original model - key (str): module name to be replaced - """ - module = model - name_list = op_name.split(".") - for name in name_list: - if hasattr(module, name): - module = getattr(module, name) - else: - module = module - return module - -def clean_cache(model): - for _, module in model.named_modules(): - if "Attention" in module.__class__.__name__: - module.h2o_kv_cache.clean_scores() - -def generate(model, **kwargs): - max_length = kwargs['max_new_tokens'] if kwargs.get('max_new_tokens') else kwargs['max_length'] - for _, module in model.named_modules(): - if "Attention" in module.__class__.__name__: - module.is_gen = True - if module.h2o_kv_cache.heavy_budget is None: - module.h2o_kv_cache.heavy_budget = int(max_length * module.h2o_kv_cache.heavy_ratio) - if module.h2o_kv_cache.recent_budget is None: - module.h2o_kv_cache.recent_budget = int(max_length * module.h2o_kv_cache.recent_ratio) - result = model.ori_generate(**kwargs) - clean_cache(model) - return result +import torch.nn as nn +from .base import KVPruner, PruneConfig def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=None): @@ -149,6 +85,28 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights, local=Tru return mask_bottom +class H2OConfig(PruneConfig): + def __init__( + self, + heavy_ratio: float = None, + recent_ratio: float = None, + heavy_budget: int = None, + recent_budget: int = None, + h2o_min_seqlen: int = -1, + real_drop: bool = True, + mean: bool = False, + local: bool = True + ): + super().__init__() + self.heavy_ratio = heavy_ratio + self.recent_ratio = recent_ratio + self.heavy_budget = heavy_budget + self.recent_budget = recent_budget + self.h2o_min_seqlen = h2o_min_seqlen + self.real_drop = real_drop + self.mean = mean + self.local = local + class H2OKVCache: def __init__( @@ -157,7 +115,8 @@ def __init__( recent_ratio=0.2, heavy_budget=None, recent_budget=None, - min_seqlen=-1 + min_seqlen=-1, + mean=False ): ## bsz, num_heads, seq_len, head_dim self.heavy_ratio = heavy_ratio @@ -166,12 +125,10 @@ def __init__( self.recent_budget = recent_budget self.hh_score = None self.min_seqlen = min_seqlen + self.mean = mean self.idx = 0 - self._past_length = 0 - - - def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): + def __call__(self, attn_score, key_states, value_states, **kwargs): seq_len = key_states.size(-2) if self.heavy_budget is None: self.heavy_budget = int(self.heavy_ratio * seq_len) @@ -184,7 +141,7 @@ def __call__(self, attn_score, key_states, value_states, mean=False, **kwargs): # attn_score shape (bsz, num_heads, seq_len, head_dim) if len(attn_score.shape) == 3: attn_score = attn_score.unsqueeze(0) - self._update_hh_score(attn_score, mean=mean) + self._update_hh_score(attn_score, mean=self.mean) # hh-selection mask = torch.zeros(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) @@ -232,34 +189,55 @@ def _update_hh_score(self, attn_score_cache, mean=False): def clean_scores(self): self.idx = 0 - self.past_length = 0 self.hh_score = None - @property - def past_length(self): - return self._past_length - - @past_length.setter - def past_length(self, value): - self._past_length = value -class H2OConfig(dict): - def __init__( - self, - heavy_ratio: float = None, - recent_ratio: float = None, - heavy_budget: int = None, - recent_budget: int = None, - h2o_min_seqlen: int = -1, - real_drop: bool = True, - mean: bool = False, - local: bool = True - ): - self.heavy_ratio = heavy_ratio - self.recent_ratio = recent_ratio - self.heavy_budget = heavy_budget - self.recent_budget = recent_budget - self.h2o_min_seqlen = h2o_min_seqlen - self.real_drop = real_drop - self.mean = mean - self.local = local +class H2OKVPruner(KVPruner): + def __init__(self, config: H2OConfig) -> None: + self.config = config + self.real_drop = self.config.real_drop + + + def self_attn_init(self, module): + module.h2o_kv_cache = H2OKVCache( + self.config.heavy_ratio, + self.config.recent_ratio, + self.config.heavy_budget, + self.config.recent_budget, + self.config.h2o_min_seqlen, + self.config.mean + ) + + def before_generate(self, model, **kwargs): + self.past_length = 0 + max_length = kwargs['max_new_tokens'] if kwargs.get('max_new_tokens') else kwargs['max_length'] + for _, module in model.named_modules(): + if "Attention" in module.__class__.__name__: + if module.h2o_kv_cache.heavy_budget is None: + module.h2o_kv_cache.heavy_budget = int(max_length * module.h2o_kv_cache.heavy_ratio) + if module.h2o_kv_cache.recent_budget is None: + module.h2o_kv_cache.recent_budget = int(max_length * module.h2o_kv_cache.recent_ratio) + + def after_generate(self, model, **kwargs): + for _, module in model.named_modules(): + if "Attention" in module.__class__.__name__: + module.h2o_kv_cache.clean_scores() + + def prune(self, module, query_states, key_states, value_states, causal_mask=None, **kwargs): + attn_weights = torch.matmul(query_states, key_states.transpose(-2, -1)) / math.sqrt(module.head_dim) + if causal_mask is not None: # no matter the length, we just slice it + attn_weights = attn_weights + causal_mask + if not self.config.real_drop: + module.h2o_kv_cache.clean_scores() + return module.h2o_kv_cache(attn_weights, key_states, value_states, **kwargs) + + def get_mask(self, module, query_states, key_states, value_states, causal_mask=None, **kwargs): + attn_weights = torch.matmul(query_states, key_states.transpose(-2, -1)) / math.sqrt(module.head_dim) + if causal_mask is not None: # no matter the length, we just slice it + attn_weights = attn_weights + causal_mask + mask = get_hh_mask( + self.config.heavy_ratio, + self.config.recent_ratio, + attn_weights, + local=self.config.local) + return mask \ No newline at end of file From 4884b3aad969685e6600738b9b4e1a72a9113d67 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Jun 2024 07:37:51 +0000 Subject: [PATCH 46/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../models/modeling_llama.py | 10 +++++----- .../kv_cache_compression/prune/__init__.py | 16 +++++++++++++++- .../kv_cache_compression/prune/base.py | 18 ++++++++++++++++-- .../modeling/kv_cache_compression/prune/h2o.py | 12 ++++++------ 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py index fd98d194667..3a961b9c1e0 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py @@ -137,10 +137,10 @@ def __init__( self._init_rope() self._init_func = [] - + def register_init_func(self, func): self._init_func.append(func) - + def post_init(self): for func in self._init_func: func(self) @@ -690,13 +690,13 @@ def __init__( self.model.layers[layer_idx].self_attn.post_init() self.model.layers[layer_idx].self_attn.pruner = self.pruner - + # Initialize weights and apply final processing self.post_init() def _generate(**kwargs): - self.pruner.before_generate(self, **kwargs) - result = self.ori_generate(**kwargs) + self.pruner.before_generate(self, **kwargs) + result = self.ori_generate(**kwargs) self.pruner.after_generate(self, **kwargs) return result diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py index 33a5058953d..7f9ede223e8 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py @@ -1,2 +1,16 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .base import PruneConfig, KVPruner -from .h2o import H2OConfig, H2OKVPruner \ No newline at end of file +from .h2o import H2OConfig, H2OKVPruner diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py index e4862c3aa77..46d9f588ba5 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + class PruneConfig(dict): def __init__(self, real_drop=True): self.real_drop = real_drop @@ -24,7 +38,7 @@ def get_mask(self, model, **kwargs): @property def past_length(self): return self._past_length - + @past_length.setter def past_length(self, value): - self._past_length = value \ No newline at end of file + self._past_length = value diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py index e3de4a605da..fe79c0bfd7c 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py @@ -196,8 +196,8 @@ class H2OKVPruner(KVPruner): def __init__(self, config: H2OConfig) -> None: self.config = config self.real_drop = self.config.real_drop - - + + def self_attn_init(self, module): module.h2o_kv_cache = H2OKVCache( self.config.heavy_ratio, @@ -207,7 +207,7 @@ def self_attn_init(self, module): self.config.h2o_min_seqlen, self.config.mean ) - + def before_generate(self, model, **kwargs): self.past_length = 0 max_length = kwargs['max_new_tokens'] if kwargs.get('max_new_tokens') else kwargs['max_length'] @@ -222,7 +222,7 @@ def after_generate(self, model, **kwargs): for _, module in model.named_modules(): if "Attention" in module.__class__.__name__: module.h2o_kv_cache.clean_scores() - + def prune(self, module, query_states, key_states, value_states, causal_mask=None, **kwargs): attn_weights = torch.matmul(query_states, key_states.transpose(-2, -1)) / math.sqrt(module.head_dim) if causal_mask is not None: # no matter the length, we just slice it @@ -230,7 +230,7 @@ def prune(self, module, query_states, key_states, value_states, causal_mask=None if not self.config.real_drop: module.h2o_kv_cache.clean_scores() return module.h2o_kv_cache(attn_weights, key_states, value_states, **kwargs) - + def get_mask(self, module, query_states, key_states, value_states, causal_mask=None, **kwargs): attn_weights = torch.matmul(query_states, key_states.transpose(-2, -1)) / math.sqrt(module.head_dim) if causal_mask is not None: # no matter the length, we just slice it @@ -240,4 +240,4 @@ def get_mask(self, module, query_states, key_states, value_states, causal_mask=N self.config.recent_ratio, attn_weights, local=self.config.local) - return mask \ No newline at end of file + return mask From 812a838b2f65438744fe1f51bd975e28382c8b05 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 24 Jun 2024 02:26:31 -0400 Subject: [PATCH 47/62] support for gaudi Signed-off-by: n1ck-guo --- .../modeling/kv_cache_compression/__init__.py | 8 - .../models/modeling_bloom.py | 410 ------ .../models/modeling_gaudi_llama.py | 1234 +++++++++++++++++ .../models/modeling_gpt_neox.py | 682 --------- .../models/modeling_mistral.py | 880 ------------ .../models/modeling_mixtral.py | 895 ------------ .../models/modeling_opt.py | 621 --------- .../kv_cache_compression/prune/base.py | 5 +- .../kv_cache_compression/prune/h2o.py | 16 +- 9 files changed, 1251 insertions(+), 3500 deletions(-) delete mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py create mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py delete mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py delete mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py delete mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py delete mode 100644 intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py index 471a68ace95..1e8078e9b40 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py @@ -14,11 +14,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from .h2o import H2OConfig -from .models.modeling_llama import H2OLlamaForCausalLM -from .models.modeling_bloom import H2OBloomForCausalLM -from .models.modeling_gpt_neox import H2OGPTNeoXForCausalLM -from .models.modeling_opt import H2OOPTForCausalLM -from .models.modeling_mistral import H2OMistralForCausalLM -from .models.modeling_mixtral import H2OMixtralForCausalLM diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py deleted file mode 100644 index 07cf8500096..00000000000 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_bloom.py +++ /dev/null @@ -1,410 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import math -import warnings -from torch.nn import functional as F - -from typing import List, Optional, Tuple, Union - -import torch -import torch.nn as nn -from torch.nn import CrossEntropyLoss -from transformers.utils import logging -from transformers.models.bloom import ( - BloomConfig, - BloomModel, - BloomPreTrainedModel, - BLOOM_INPUTS_DOCSTRING, - _CHECKPOINT_FOR_DOC, - _CONFIG_FOR_DOC -) -from transformers.file_utils import ( - add_code_sample_docstrings, - add_start_docstrings_to_model_forward - ) -from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions - -from ..h2o import H2OKVCache, H2OConfig - -logger = logging.get_logger(__name__) - -class H2OBloomAttention(nn.Module): - def __init__( - self, - config: BloomConfig, - h2o_config: H2OConfig - ): - super().__init__() - - self.pretraining_tp = config.pretraining_tp - self.slow_but_exact = config.slow_but_exact - - self.hidden_size = config.hidden_size - self.num_heads = config.n_head - self.head_dim = self.hidden_size // self.num_heads - self.split_size = self.hidden_size - self.hidden_dropout = config.hidden_dropout - - if self.head_dim * self.num_heads != self.hidden_size: - raise ValueError( - f"`hidden_size` must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:" - f" {self.num_heads})." - ) - - # Layer-wise attention scaling - self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim) - self.beta = 1.0 - - self.query_key_value = nn.Linear(self.hidden_size, 3 * self.hidden_size, bias=True) - self.dense = nn.Linear(self.hidden_size, self.hidden_size) - self.attention_dropout = nn.Dropout(config.attention_dropout) - - # for h2o - if real_drop: - real_drop = False - logger.warning_once("BloomAttention not support for kv cache, usning simulation mode.") - self.h2o_config = h2o_config - self.is_gen = False - self.mean = h2o_config.mean - self.local = h2o_config.local - - self.heavy_ratio = h2o_config.heavy_ratio - self.recent_ratio = h2o_config.recent_ratio - self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) - - def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory - storage as `fused_qkv` - - Args: - fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim] - - Returns: - query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim] - value: [batch_size, seq_length, num_heads, head_dim] - """ - batch_size, seq_length, three_times_hidden_size = fused_qkv.shape - fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads, 3, self.head_dim) - return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :] - - def _merge_heads(self, x: torch.Tensor) -> torch.Tensor: - """Merge heads together over the last dimension. - - Args: - x (`torch.tensor`, *required*): [batch_size * num_heads, seq_length, head_dim] - - Returns: - torch.tensor: [batch_size, seq_length, num_heads * head_dim] - """ - # What we want to achieve is: - # batch_size * num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads * head_dim - batch_size_and_num_heads, seq_length, _ = x.shape - batch_size = batch_size_and_num_heads // self.num_heads - - # First view to decompose the batch size - # batch_size * num_heads, seq_length, head_dim -> batch_size, num_heads, seq_length, head_dim - x = x.view(batch_size, self.num_heads, seq_length, self.head_dim) - - # batch_size, num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads, head_dim - x = x.permute(0, 2, 1, 3) - - # batch_size, seq_length, num_heads, head_dim -> batch_size, seq_length, num_heads * head_dim - return x.reshape(batch_size, seq_length, self.num_heads * self.head_dim) - - def forward( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - alibi: torch.Tensor, - attention_mask: torch.Tensor, - layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - head_mask: Optional[torch.Tensor] = None, - use_cache: bool = False, - output_attentions: bool = False, - ): - fused_qkv = self.query_key_value(hidden_states) # [batch_size, seq_length, 3 x hidden_size] - - # 3 x [batch_size, seq_length, num_heads, head_dim] - (query_layer, key_layer, value_layer) = self._split_heads(fused_qkv) - - batch_size, q_length, _, _ = query_layer.shape - - query_layer = query_layer.transpose(1, 2).reshape(batch_size * self.num_heads, q_length, self.head_dim) - key_layer = key_layer.permute(0, 2, 3, 1).reshape(batch_size * self.num_heads, self.head_dim, q_length) - value_layer = value_layer.transpose(1, 2).reshape(batch_size * self.num_heads, q_length, self.head_dim) - if layer_past is not None: - past_key, past_value = layer_past - # concatenate along seq_length dimension: - # - key: [batch_size * self.num_heads, head_dim, kv_length] - # - value: [batch_size * self.num_heads, kv_length, head_dim] - key_layer = torch.cat((past_key, key_layer), dim=2) - value_layer = torch.cat((past_value, value_layer), dim=1) - - _, _, kv_length = key_layer.shape - - if use_cache is True: - present = (key_layer, value_layer) - else: - present = None - - # [batch_size * num_heads, q_length, kv_length] - # we use `torch.Tensor.baddbmm` instead of `torch.baddbmm` as the latter isn't supported by TorchScript v1.11 - matmul_result = alibi.baddbmm( - batch1=query_layer, - batch2=key_layer, - beta=self.beta, - alpha=self.inv_norm_factor, - ) - - # change view to [batch_size, num_heads, q_length, kv_length] - attention_scores = matmul_result.view(batch_size, self.num_heads, q_length, kv_length) - - input_dtype = attention_scores.dtype - # `float16` has a minimum value of -65504.0, whereas `bfloat16` and `float32` have a minimum value of `-3.4e+38` - if input_dtype == torch.float16: - attention_scores = attention_scores.to(torch.float) - attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.finfo(attention_scores.dtype).min) - - # get hh mask - from ..h2o import get_hh_mask - mask = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights.detach().clone(), local=self.local) - attn_weights[~mask] = torch.finfo(attn_weights.dtype).min - - attention_probs = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(input_dtype) - # [batch_size, num_heads, q_length, kv_length] - attention_probs = self.attention_dropout(attention_probs) - - if head_mask is not None: - attention_probs = attention_probs * head_mask - - # change view [batch_size x num_heads, q_length, kv_length] - attention_probs_reshaped = attention_probs.view(batch_size * self.num_heads, q_length, kv_length) - - # matmul: [batch_size * num_heads, q_length, head_dim] - context_layer = torch.bmm(attention_probs_reshaped, value_layer) - - # change view [batch_size, q_length, num_heads * head_dim] - context_layer = self._merge_heads(context_layer) - - # aggregate results across tp ranks. See here: https://github.com/pytorch/pytorch/issues/76232 - if self.pretraining_tp > 1 and self.slow_but_exact: - slices = self.hidden_size / self.pretraining_tp - output_tensor = torch.zeros_like(context_layer) - for i in range(self.pretraining_tp): - output_tensor = output_tensor + F.linear( - context_layer[:, :, int(i * slices) : int((i + 1) * slices)], - self.dense.weight[:, int(i * slices) : int((i + 1) * slices)], - ) - else: - output_tensor = self.dense(context_layer) - - output_tensor = dropout_add(output_tensor, residual, self.hidden_dropout, self.training) - - outputs = (output_tensor, present) - if output_attentions: - outputs += (attention_probs,) - - return outputs - -def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor: - """Dropout add function. - - Args: - x (`torch.tensor`, *required*): - input tensor - residual (`torch.tensor`, *required*): - residual tensor - prob (`float`, *required*): - dropout probability - training (`bool`, *required*): - training mode - """ - out = F.dropout(x, p=prob, training=training) - out = residual + out - return out - -class H2OBloomForCausalLM(BloomPreTrainedModel): - _tied_weights_keys = ["lm_head.weight"] - - def __init__( - self, - config: BloomConfig, - h2o_config: H2OConfig - ): - super().__init__(config) - self.transformer = BloomModel(config) - self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - - num_layers = len(self.transformer.h) - for layer_idx in range(num_layers): - self.transformer.h[layer_idx].self_attention = H2OBloomAttention( - config, - h2o_config - ) - # Initialize weights and apply final processing - self.post_init() - - def get_output_embeddings(self): - return self.lm_head - - def set_output_embeddings(self, new_embeddings: torch.Tensor): - self.lm_head = new_embeddings - - def prepare_inputs_for_generation( - self, - input_ids: torch.LongTensor, - past_key_values: Optional[torch.Tensor] = None, - attention_mask: Optional[torch.Tensor] = None, - inputs_embeds: Optional[torch.Tensor] = None, - **kwargs, - ) -> dict: - # only last tokens for input_ids if past is not None - if past_key_values is not None: - past_length = past_key_values[0][0].shape[2] - - # Some generation methods already pass only the last input ID - if input_ids.shape[1] > past_length: - remove_prefix_length = past_length - else: - # Default to old behavior: keep only final ID - remove_prefix_length = input_ids.shape[1] - 1 - - input_ids = input_ids[:, remove_prefix_length:] - - # the cache may be in the standard format (e.g. in contrastive search), convert to bloom's format if needed - if past_key_values[0][0].shape[0] == input_ids.shape[0]: - past_key_values = self._convert_to_bloom_cache(past_key_values) - - # if `inputs_embeds` are passed, we only want to use them in the 1st generation step - if inputs_embeds is not None and past_key_values is None: - model_inputs = {"inputs_embeds": inputs_embeds} - else: - model_inputs = {"input_ids": input_ids} - - model_inputs.update( - { - "past_key_values": past_key_values, - "use_cache": kwargs.get("use_cache"), - "attention_mask": attention_mask, - } - ) - return model_inputs - - @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, - attention_mask: Optional[torch.Tensor] = None, - head_mask: Optional[torch.Tensor] = None, - inputs_embeds: Optional[torch.Tensor] = None, - labels: Optional[torch.Tensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - **deprecated_arguments, - ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: - r"""Labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - - Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set - `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` - are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - """ - if deprecated_arguments.pop("position_ids", False) is not False: - # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` - warnings.warn( - "`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore" - " passing `position_ids`.", - FutureWarning, - ) - if len(deprecated_arguments) > 0: - raise ValueError(f"Got unexpected arguments: {deprecated_arguments}") - - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - transformer_outputs = self.transformer( - input_ids, - past_key_values=past_key_values, - attention_mask=attention_mask, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - hidden_states = transformer_outputs[0] - - lm_logits = self.lm_head(hidden_states) - - loss = None - if labels is not None: - # move labels to correct device to enable model parallelism - labels = labels.to(lm_logits.device) - # Shift so that tokens < n predict n - shift_logits = lm_logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - batch_size, seq_length, vocab_size = shift_logits.shape - # Flatten the tokens - loss_fct = CrossEntropyLoss() - loss = loss_fct( - shift_logits.view(batch_size * seq_length, vocab_size), shift_labels.view(batch_size * seq_length) - ) - - if not return_dict: - output = (lm_logits,) + transformer_outputs[1:] - return ((loss,) + output) if loss is not None else output - - return CausalLMOutputWithCrossAttentions( - loss=loss, - logits=lm_logits, - past_key_values=transformer_outputs.past_key_values, - hidden_states=transformer_outputs.hidden_states, - attentions=transformer_outputs.attentions, - ) - - def _reorder_cache( - self, past: Tuple[Tuple[torch.Tensor, torch.Tensor], ...], beam_idx: torch.LongTensor - ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]: - """This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or - [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct - beam_idx at every generation step. - - Output shares the same memory storage as `past`. - """ - standardized_past = self._convert_to_standard_cache(past, batch_size=len(beam_idx)) - - # Get a copy of `beam_idx` on all the devices where we need those indices. - device_to_beam_idx = { - past_state.device: beam_idx.to(past_state.device) for layer_past in past for past_state in layer_past - } - reordered_past = tuple( - ( - layer_past[0].index_select(0, device_to_beam_idx[layer_past[0].device]), - layer_past[1].index_select(0, device_to_beam_idx[layer_past[0].device]), - ) - for layer_past in standardized_past - ) - return self._convert_to_bloom_cache(reordered_past) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py new file mode 100644 index 00000000000..ac2cba82bcf --- /dev/null +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py @@ -0,0 +1,1234 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import warnings +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers.cache_utils import Cache, DynamicCache, StaticCache # pylint: disable=E0611 +from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import ( + LlamaAttention, + LlamaDecoderLayer, + LlamaForCausalLM, + LlamaMLP, + LlamaModel, + LlamaRMSNorm, + LlamaPreTrainedModel, + apply_rotary_pos_emb, + logger, +) + +from ...modeling_gaudi.models.modeling_attn_mask_utils import( + _gaudi_prepare_4d_causal_attention_mask, +) + + +try: + from habana_frameworks.torch.hpex.kernels import RotaryPosEmbeddingHelperV2 as FusedRoPE + + has_fused_rope = True +except ImportError: + has_fused_rope = False + print("Not using HPU fused kernel for apply_rotary_pos_emb") + +try: + from habana_frameworks.torch.hpex.normalization import FusedRMSNorm as FusedRMSNorm + + has_fused_rms_norm = True +except ImportError: + has_fused_rms_norm = False + print("Not using HPU fused kernel for RMSNorm") + +try: + from habana_frameworks.torch.hpex.kernels import FusedSDPA +except ImportError: + print("Not using HPU fused scaled dot-product attention kernel.") + FusedSDPA = None + +import habana_frameworks.torch.core as htcore # pylint: disable=E0401 + +from ..prune import PruneConfig, H2OConfig + + +def gaudi_llama_rmsnorm_forward(self, hidden_states): + """ + + The only differences are: + - override RMSNorm with Habana fused RMSNorm + """ + if hidden_states.device.type == "hpu" and has_fused_rms_norm: + # mixed dtypes are not good for FusedRMSNorm, both inputs need to have same dtype + if hidden_states.dtype != self.weight.dtype: + orig_dtype = hidden_states.dtype + hidden_states = FusedRMSNorm.apply(hidden_states.to(self.weight.dtype), self.weight, self.variance_epsilon) + return hidden_states.to(orig_dtype) + else: + hidden_states = FusedRMSNorm.apply(hidden_states, self.weight, self.variance_epsilon) + return hidden_states + else: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +class GaudiLlamaMLP(LlamaMLP): + def pre_mlp_forward(self, x): + if self.config.pretraining_tp > 1: + slice = self.intermediate_size // self.config.pretraining_tp + gate_proj_slices = self.gate_proj.weight.split(slice, dim=0) + up_proj_slices = self.up_proj.weight.split(slice, dim=0) + down_proj_slices = self.down_proj.weight.split(slice, dim=1) + + gate_proj = torch.cat( + [F.linear(x, gate_proj_slices[i]) \ + for i in range(self.config.pretraining_tp)], dim=-1) # pylint: disable=E1102 + up_proj = torch.cat([F.linear(x, up_proj_slices[i]) \ + for i in range(self.config.pretraining_tp)], dim=-1) # pylint: disable=E1102 + + intermediate_states = (self.act_fn(gate_proj) * up_proj).split(slice, dim=2) + down_proj = [ + F.linear(intermediate_states[i], down_proj_slices[i]) \ + for i in range(self.config.pretraining_tp)] # pylint: disable=E1102 + output = sum(down_proj) + else: + input = self.act_fn(self.gate_proj(x)) * self.up_proj(x) + output = self.down_proj(input) + return output + + def mlp_all_reduce(self, x): + if hasattr(self.down_proj, "all_reduce"): + self.down_proj.all_reduce(x) + + def post_mlp_forward(self, x): + if self.config.pretraining_tp > 1: + return x + if hasattr(self.down_proj, "post_all_reduce"): + return self.down_proj.post_all_reduce(x) + return x + + +def gaudi_llama_repeat_kv( + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + attention_mask: torch.Tensor, + n_rep: int, +): + """ + + The only differences are: + - Append num_key_value_heads == 1 check as kv states can be broadcasted during + matmuls so need to expand and reshape them. + - Add new args query_states, key_states, value_states and attention_mask and + update the logic for expansion. + The query states go from (batch, num_heads, seqlen, head_dim) to + (batch, num_key_value_heads, n_rep, seqlen, head_dim) + The key/value states go from (batch, num_key_value_heads, seqlen, head_dim) to + (batch, num_key_value_heads, 1, seqlen, head_dim) + """ + batch, num_key_value_heads, kv_len, head_dim = key_states.shape + if n_rep == 1 or num_key_value_heads == 1: + return query_states, key_states, value_states, attention_mask + + new_kv_shape = (batch, num_key_value_heads, 1, kv_len, head_dim) + key_states = key_states.reshape(new_kv_shape) + value_states = value_states.reshape(new_kv_shape) + + batch, _, q_len, head_dim = query_states.shape + new_q_shape = (batch, num_key_value_heads, n_rep, q_len, head_dim) + query_states = query_states.reshape(new_q_shape) + + if attention_mask is not None: + # Add groups dim and set to 1 + attention_mask = attention_mask.unsqueeze(1) + + return query_states, key_states, value_states, attention_mask + + +class Matmul(torch.nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x, y): + return torch.matmul(x, y) + + +class KVCache(torch.nn.Module): + def __init__(self): + super(KVCache, self).__init__() + self.cache = None + self.inp_seq_len = -1 + + def allocate(self, inp_seq_len, dtype, device, shape): + if self.cache is None or self.cache.shape != shape: + self.inp_seq_len = inp_seq_len + self.cache = torch.zeros(shape, dtype=dtype, device=device) + else: + assert ( + self.inp_seq_len == inp_seq_len + ), f"inp_seq_len must be the same. self.inp_seq_len:{self.inp_seq_len} inp_seq_len:{inp_seq_len}" + self.cache.fill_(0) + + def update(self, prev, cur, dim, idx, inp_seq_len): + orig_cur = cur + if prev.shape == cur.shape: + prev.copy_(cur) + return orig_cur + if cur.shape[2] > 1 and cur.shape[2] <= prev.shape[2]: + # Initialize + prev[:, :, :inp_seq_len, :].copy_(cur) + return orig_cur + assert cur.shape[2] == 1, f"Cannot update kv-cache. Unsupported shapes. prev:{prev.shape} cur:{cur.shape}" + if idx is not None: + prev.index_copy_(dim, idx - 1, cur) + return prev + else: + return torch.cat((prev, cur), dim=dim) + + def get_shape(self): + if self.cache is None: + return None + return self.cache.shape + + def forward(self, cur, dim, idx): + return self.update(self.cache, cur, dim, idx, self.inp_seq_len) + + +class GaudiLlamaRotaryEmbedding(torch.nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): + super().__init__() + + self.scaling_factor = scaling_factor + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + # Build here to make `torch.jit.trace` work. + self._set_cos_sin_cache( + seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype() + ) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) + + freqs = torch.outer(t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("_cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("_sin_cached", emb.sin().to(dtype), persistent=False) + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if seq_len > self.max_seq_len_cached: + self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype) + + return ( + self._cos_cached[:seq_len].to(dtype=x.dtype), + self._sin_cached[:seq_len].to(dtype=x.dtype), + ) + + +class GaudiLlamaLinearScalingRotaryEmbedding(GaudiLlamaRotaryEmbedding): + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) + t = t / self.scaling_factor + + freqs = torch.outer(t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("_cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("_sin_cached", emb.sin().to(dtype), persistent=False) + + +class GaudiLlamaDynamicNTKScalingRotaryEmbedding(GaudiLlamaRotaryEmbedding): + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + + if seq_len > self.max_position_embeddings: + base = self.base * ( + (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1) + ) ** (self.dim / (self.dim - 2)) + inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) + + freqs = torch.outer(t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("_cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("_sin_cached", emb.sin().to(dtype), persistent=False) + + +class GaudiLlamaAttention(LlamaAttention): + def __init__(self, config: LlamaConfig, layer_idx: Optional[int] = None): + super().__init__(config, layer_idx) + + self.matmul_qk = Matmul() + self.matmul_av = Matmul() + self.k_cache = KVCache() + self.v_cache = KVCache() + self.inp_seq_len = -1 + self.norm_factor = 1.0 / math.sqrt(self.head_dim) + + self._init_func = [] + + def register_init_func(self, func): + self._init_func.append(func) + + def post_init(self): + for func in self._init_func: + func(self) + + def allocate_kv_cache(self, batch_size, max_seq_len, inp_seq_len): + cache_shape = (batch_size, self.num_key_value_heads, max_seq_len, self.head_dim) + device = self.k_proj.weight.device + dtype = self.config.torch_dtype + self.k_cache.allocate(inp_seq_len, dtype, device, cache_shape) + self.v_cache.allocate(inp_seq_len, dtype, device, cache_shape) + + def update_sincos_cache(self, seq_len): + # Call rotary emb forward() to update cos/sin cache when inferring more than self.max_position_embeddings + # This helps in avoiding creation of these caches during actual model forward pass and + # reduce memory consumption and improve performance. + if seq_len > self.max_position_embeddings: # pylint: disable=E0203 + self.max_position_embeddings = seq_len + _, _ = self.rotary_emb(self.k_proj.weight, seq_len=seq_len) + + def reorder(self, tensor, beam_idx, dim_a, dim_b): + updated = tensor.index_select(0, beam_idx) + tensor.copy_(updated) + + def reorder_kv_cache(self, beam_idx: torch.LongTensor): + if self.k_cache.cache is None: + return (None, None) + + head_dim = self.k_cache.cache.size(-1) + seq_length = self.k_cache.cache.size(-2) + self.reorder(self.k_cache.cache, beam_idx, seq_length, head_dim) + self.reorder(self.v_cache.cache, beam_idx, seq_length, head_dim) + return (self.k_cache.cache.shape, self.v_cache.cache.shape) + + def pre_attn_forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + token_idx: Optional[torch.Tensor] = None, + attn_softmax_bf16: Optional[bool] = False, + reuse_cache: Optional[bool] = False, + use_flash_attention: Optional[bool] = False, + flash_attention_recompute: Optional[bool] = False, + flash_attention_causal_mask: Optional[bool] = False, + cache_idx: int = None, + cache_prune_num: int = 0, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """ + + The only differences are: + - add new args token_idx + - optimize KV cache + - add new args attn_softmax_bf16 + - add new args reuse_cache + - add new args use_flash_attention + - add new arg flash_attention_recompute + - add new arg flash_attention_causal_mask + - add new arg cache_prune_num for attention_sinks + """ + bsz, q_len, _ = hidden_states.size() + + if self.config.pretraining_tp > 1: + key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp + query_slices = self.q_proj.weight.split( + (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 + ) + key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) + value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) + + query_states = [F.linear(hidden_states, query_slices[i]) \ + for i in range(self.config.pretraining_tp)] # pylint: disable=E1102 + query_states = torch.cat(query_states, dim=-1) + + key_states = [F.linear(hidden_states, key_slices[i]) \ + for i in range(self.config.pretraining_tp)] # pylint: disable=E1102 + key_states = torch.cat(key_states, dim=-1) + + value_states = [F.linear(hidden_states, value_slices[i]) \ + for i in range(self.config.pretraining_tp)] # pylint: disable=E1102 + value_states = torch.cat(value_states, dim=-1) + + else: + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if token_idx is None: + if hasattr(past_key_value, "get_usable_length"): + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + else: + kv_seq_len += past_key_value[0].shape[-2] + else: + if reuse_cache: + kv_seq_len = past_key_value[0][-2] + else: + kv_seq_len = past_key_value[0].shape[-2] + + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_customized_rope(query_states, key_states, cos, sin, position_ids) + + if use_cache: + # reuse k, v, self_attention + if reuse_cache: + # key_states = self.k_cache(key_states, 2, token_idx) + # value_states = self.v_cache(value_states, 2, token_idx) + # past_key_value = (self.k_cache.get_shape(), self.v_cache.get_shape()) + + # pruning kv cache + if self.pruner.real_drop: + if self.layer_idx == 0: + self.pruner.past_length += query_states.size(-2) + new_key_states, new_value_states = self.pruner.prune( + self, + query_states, + key_states, + value_states, + causal_mask=causal_mask + ) + key_states = self.k_cache(new_key_states, 2, token_idx) + value_states = self.v_cache(new_value_states, 2, token_idx) + past_key_value = (self.k_cache.get_shape(), self.v_cache.get_shape()) + else: + if past_key_value is None: + past_key = torch.zeros(key_states.shape, dtype=self.k_proj.weight.dtype, device=key_states.device) + past_value = torch.zeros( + key_states.shape, dtype=self.k_proj.weight.dtype, device=key_states.device + ) + past_key_value = (past_key, past_value) + key_states = self.k_cache.update(past_key_value[0], key_states, 2, token_idx, self.inp_seq_len) + value_states = self.v_cache.update(past_key_value[1], value_states, 2, token_idx, self.inp_seq_len) + if token_idx is None: + past_key_value = (key_states, value_states) + + if cache_idx is not None and q_len == 1: + key_states = key_states[:, :, :cache_idx, :] + value_states = value_states[:, :, :cache_idx, :] + if attention_mask is not None: + attention_mask = attention_mask[:, :, :, :cache_idx] + kv_seq_len = key_states.shape[-2] + else: + past_key_value = None + + if use_flash_attention and FusedSDPA: + import habana_frameworks.torch.hpu as ht # pylint: disable=E0401 + + if q_len == 1: + # next token + with ht.sdp_kernel(enable_recompute=False): + attn_output = FusedSDPA.apply( + query_states, key_states, value_states, attention_mask, 0.0, False, None + ) + else: + # first token + if flash_attention_causal_mask: + # causal masking on first token requires inputs to be of the same length + with ht.sdp_kernel(enable_recompute=flash_attention_recompute): + attn_output = FusedSDPA.apply(query_states, key_states, value_states, None, 0.0, True, None) + else: + with ht.sdp_kernel(enable_recompute=flash_attention_recompute): + attn_output = FusedSDPA.apply( + query_states, key_states, value_states, attention_mask, 0.0, False, None + ) + + else: + query_states, key_states, value_states, attention_mask = gaudi_llama_repeat_kv( + query_states, key_states, value_states, attention_mask, self.num_key_value_groups + ) + + attn_weights = self.matmul_qk(query_states, key_states.transpose(-2, -1)) * self.norm_factor + + if not self.pruner.real_drop: + mask = self.pruner.get_mask(self, query_states, key_states, value_states, + causal_mask=causal_mask) + attn_weights[~mask] = torch.finfo(attn_weights.dtype).min + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask + if cache_position is not None: + causal_mask = attention_mask[:, :, cache_position, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + if attn_softmax_bf16: + attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1, dtype=query_states.dtype) + else: + # upcast attention to fp32 + attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to( + query_states.dtype + ) + attn_weights = torch.nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = self.matmul_av(attn_weights, value_states) + attn_output = attn_output.reshape(bsz, -1, q_len, self.head_dim) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + def attention_all_reduce(self, attn_output): + if hasattr(self.o_proj, "all_reduce"): + self.o_proj.all_reduce(attn_output) + + def post_attn_forward(self, attn_output): + if hasattr(self.o_proj, "post_all_reduce"): + self.o_proj.post_all_reduce(attn_output) + return attn_output + + +class GaudiLlamaDecoderLayer(LlamaDecoderLayer): + def __init__(self, config: LlamaConfig, layer_idx: int): + super(GaudiLlamaDecoderLayer, self).__init__(config, layer_idx) + self.hidden_size = config.hidden_size + + self.self_attn = GaudiLlamaAttention(config=config, layer_idx=layer_idx) + + self.mlp = GaudiLlamaMLP(config) + self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def allocate_kv_cache(self, batch_size, max_seq_len, inp_seq_len): + self.self_attn.allocate_kv_cache(batch_size, max_seq_len, inp_seq_len) + + def reorder_kv_cache(self, beam_idx: torch.LongTensor): + return self.self_attn.reorder_kv_cache(beam_idx) + + def update_sincos_cache(self, seq_len): + self.self_attn.update_sincos_cache(seq_len) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + token_idx: Optional[torch.Tensor] = None, + attn_softmax_bf16: Optional[bool] = False, + reuse_cache: Optional[bool] = False, + use_flash_attention: Optional[bool] = False, + flash_attention_recompute: Optional[bool] = False, + flash_attention_causal_mask: Optional[bool] = False, + cache_idx: int = None, + cache_prune_num: int = 0, + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + + The only differences are: + - add new args token_idx + - add new args attn_softmax_bf16 + - add new args reuse_cache + - add new args use_flash_attention + - add new arg flash_attention_recompute + - add new arg flash_attention_causal_mask + - add new arg cache_prune_num for attention_sinks + """ + residual = hidden_states + hidden_states, self_attn_weights, present_key_value = self.pre_attn( + hidden_states, + attention_mask, + position_ids, + past_key_value, + output_attentions, + use_cache, + cache_position, + token_idx, + attn_softmax_bf16, + reuse_cache, + use_flash_attention=use_flash_attention, + flash_attention_recompute=flash_attention_recompute, + flash_attention_causal_mask=flash_attention_causal_mask, + cache_idx=cache_idx, + cache_prune_num = cache_prune_num, + **kwargs, + ) + self.self_attn.attention_all_reduce(hidden_states) + hidden_states, residual = self.post_attn_pre_mlp(hidden_states, residual) + self.mlp.mlp_all_reduce(hidden_states) + hidden_states = self.post_mlp(hidden_states, residual) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + if use_cache: + outputs += (present_key_value,) + + return outputs + + def pre_attn( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + token_idx: Optional[torch.Tensor] = None, + attn_softmax_bf16: Optional[bool] = False, + reuse_cache: Optional[bool] = False, + use_flash_attention: Optional[bool] = False, + flash_attention_recompute: Optional[bool] = False, + flash_attention_causal_mask: Optional[bool] = False, + cache_idx: int = None, + cache_prune_num: int = 0, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + hidden_states = self.input_layernorm(hidden_states) + hidden_states, attn_weights, present_key_value = self.self_attn.pre_attn_forward( + hidden_states, + attention_mask, + position_ids, + past_key_value, + output_attentions, + use_cache, + cache_position, + token_idx, + attn_softmax_bf16, + reuse_cache, + use_flash_attention, + flash_attention_recompute, + flash_attention_causal_mask, + cache_idx=cache_idx, + cache_prune_num = cache_prune_num, + ) + return hidden_states, attn_weights, present_key_value + + def post_attn_pre_mlp(self, hidden_states, residual): + hidden_states = self.self_attn.post_attn_forward(hidden_states) + + if self.training: + hidden_states = hidden_states + residual + residual = hidden_states + else: + residual.add_(hidden_states) + hidden_states = residual + + hidden_states = self.post_attention_layernorm(hidden_states) + + hidden_states = self.mlp.pre_mlp_forward(hidden_states) + return hidden_states, residual + + def post_mlp(self, hidden_states, residual): + hidden_states = self.mlp.post_mlp_forward(hidden_states) + + if self.training: + hidden_states = hidden_states + residual + else: + residual.add_(hidden_states) + hidden_states = residual + + return hidden_states + + +class GaudiLlamaModel(LlamaModel): + """""" + + def __init__(self, config: LlamaConfig): + """ + + 1. set fill_value to 1 instead of True + 2. add device=self.device + """ + super(GaudiLlamaModel, self).__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = torch.nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = torch.nn.ModuleList( + [GaudiLlamaDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.gradient_checkpointing = False + + # Register a causal mask to separate causal and padding mask creation. Merging happens in the attention class. + causal_mask = torch.full( + (config.max_position_embeddings, config.max_position_embeddings), + fill_value=1, + dtype=torch.bool, + ) + self.register_buffer("causal_mask", torch.triu(causal_mask, diagonal=1), persistent=False) + # Initialize weights and apply final processing + self.post_init() + + def allocate_kv_cache(self, batch_size, max_seq_len, inp_seq_len): + for layer in self.layers: + layer.allocate_kv_cache(batch_size, max_seq_len, inp_seq_len) + + def reorder_kv_cache(self, beam_idx: torch.LongTensor): + return tuple(layer.reorder_kv_cache(beam_idx) for layer in self.layers) + + def update_sincos_cache(self, seq_len): + for layer in self.layers: + layer.update_sincos_cache(seq_len) + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + token_idx: Optional[torch.Tensor] = None, + attn_softmax_bf16: Optional[bool] = False, + reuse_cache: Optional[bool] = False, + use_flash_attention: Optional[bool] = False, + flash_attention_recompute: Optional[bool] = False, + flash_attention_causal_mask: Optional[bool] = False, + cache_idx: int = None, + lazy_mode: Optional[bool] = True, + cache_prune_num: int = 0, + ) -> Union[Tuple, BaseModelOutputWithPast]: + """ + + The only differences are: + - add new args token_idx + - add new args attn_softmax_bf16 + - add new args reuse_cache + - add new args use_flash_attention + - add new arg flash_attention_recompute + - add new arg flash_attention_causal_mask + - add new arg lazy_mode + - add new arg cache_prune_num for attention_sinks + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError( + "You cannot specify both input_ids and inputs_embeds at the same time, and must specify either one" + ) + elif input_ids is not None: + batch_size, seq_length = input_ids.shape[:2] + elif inputs_embeds is not None: + batch_size, seq_length = inputs_embeds.shape[:2] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + if self.gradient_checkpointing and self.training and use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`." + ) + use_cache = False + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + ignore_cache_position = True # Ignoring cache position for HPU + use_new_cache = False # Ignoring new Cache path for HPU + past_seen_tokens = 0 + + if past_key_values is not None and use_cache: # kept for BC (cache positions) + if reuse_cache: + past_seen_tokens = past_key_values[0][0][2] + else: + if use_new_cache: + if not isinstance(past_key_values, StaticCache): + past_key_values = DynamicCache.from_legacy_cache(past_key_values) + past_seen_tokens = past_key_values.get_seq_length() + else: + past_seen_tokens = past_key_values[0][0].shape[2] + + if ignore_cache_position is False: + if cache_position is None: + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + if position_ids is None and cache_position: + position_ids = cache_position.unsqueeze(0) + + else: + if position_ids is None: + position_ids = torch.arange( + past_seen_tokens, seq_length + past_seen_tokens, dtype=torch.long, device=inputs_embeds.device + ) + position_ids = position_ids.unsqueeze(0) + cache_position = None + + # HPU specific mask generation + if ignore_cache_position: + # workaround for attention_sinks attention_mask which has fixed seq_len at dim -1 + if hasattr(self, "attention_sink_size") and hasattr(self, "attention_sink_window_size"): + past_seen_tokens = self.attention_sink_size + self.attention_sink_window_size - seq_length + causal_mask = _gaudi_prepare_4d_causal_attention_mask( + attention_mask, + input_ids.shape if input_ids is not None else (batch_size, seq_length), + inputs_embeds, + past_seen_tokens, + ) + else: + causal_mask = self._update_causal_mask(attention_mask, inputs_embeds) + # embed positions + hidden_states = inputs_embeds + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = () if not use_new_cache else None + + if lazy_mode: + htcore.mark_step() + + for layer_idx, decoder_layer in enumerate(self.layers): + if ( + lazy_mode + and not self.training + and (torch.distributed.is_initialized() is False or torch.distributed.get_world_size() == 1) + ): + htcore.mark_step() + + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + decoder_layer.__call__, + hidden_states, + causal_mask, + position_ids, + past_key_values, + output_attentions, + use_cache, + cache_position, + None, + attn_softmax_bf16, + False, + use_flash_attention, + flash_attention_recompute, + flash_attention_causal_mask, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_value=None if past_key_values is None else past_key_values[layer_idx], + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + token_idx=token_idx, + attn_softmax_bf16=attn_softmax_bf16, + reuse_cache=reuse_cache, + use_flash_attention=use_flash_attention, + flash_attention_recompute=flash_attention_recompute, + flash_attention_causal_mask=flash_attention_causal_mask, + cache_idx=cache_idx, + cache_prune_num = cache_prune_num, + ) + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += (layer_outputs[2 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = None + if use_cache: + next_cache = ( + next_decoder_cache.to_legacy_cache() if isinstance(next_decoder_cache, Cache) else next_decoder_cache + ) + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class GaudiLlamaForCausalLM(LlamaPreTrainedModel): + """ + + The only differences are: + - add new args token_idx + - add token_idx into model_inputs + - from step2 when enable KV cache, slice next_input_ids from input_ids base on the token_idx + - from step2 when enable KV cache, slice next_position_ids from position_ids base on the token_idx + - add new args attn_softmax_bf16 + - add new args reuse_cache + - add new arg cache_prune_num for attention_sinks + """ + def __init__( + self, + config: LlamaConfig, + prune_config: PruneConfig, + ): + super().__init__(config) + self.model = LlamaModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + if isinstance(prune_config, H2OConfig): + from ..prune import H2OKVPruner + self.pruner = H2OKVPruner(prune_config) + else: + from ..prune import KVPruner + self.pruner = KVPruner(prune_config) + + num_layers = len(self.model.layers) + for layer_idx in range(num_layers): + module = self.model.layers[layer_idx].self_attn + self.model.layers[layer_idx].self_attn = GaudiLlamaAttention( + config, + layer_idx + ) + self.model.layers[layer_idx].self_attn.register_init_func(self.pruner.self_attn_init) + self.model.layers[layer_idx].self_attn.post_init() + + self.model.layers[layer_idx].self_attn.pruner = self.pruner + + # Initialize weights and apply final processing + self.post_init() + + def _generate(*args, **kwargs): + self.pruner.before_generate(self, *args, **kwargs) + result = self.ori_generate(*args, **kwargs) + self.pruner.after_generate(self,*args, **kwargs) + return result + + self.ori_generate = self.generate + self.generate = _generate + + def allocate_kv_cache(self, batch_size, max_seq_len, inp_seq_len): + self.model.allocate_kv_cache(batch_size, max_seq_len, inp_seq_len) + + def reorder_kv_cache(self, beam_idx: torch.LongTensor): + return self.model.reorder_kv_cache(beam_idx) + + def update_sincos_cache(self, seq_len): + self.model.update_sincos_cache(seq_len) + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + token_idx: Optional[torch.Tensor] = None, + trim_logits: Optional[bool] = False, + attn_softmax_bf16: Optional[bool] = False, + reuse_cache: Optional[bool] = False, + use_flash_attention: Optional[bool] = False, + flash_attention_recompute: Optional[bool] = False, + flash_attention_causal_mask: Optional[bool] = False, + cache_idx: int = None, + lazy_mode: Optional[bool] = True, + cache_prune_num: int = 0, + ) -> Union[Tuple, CausalLMOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if self.generation_config.use_fused_rope is False: + global has_fused_rope + has_fused_rope = False + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + token_idx=token_idx, + attn_softmax_bf16=attn_softmax_bf16, + reuse_cache=reuse_cache, + use_flash_attention=use_flash_attention, + flash_attention_recompute=flash_attention_recompute, + flash_attention_causal_mask=flash_attention_causal_mask, + cache_idx=cache_idx, + lazy_mode=lazy_mode, + cache_prune_num=cache_prune_num, + ) + hidden_states = outputs[0] + _, seq_len, _ = hidden_states.shape + if seq_len > 1 and trim_logits and not self.training: + if token_idx is not None: + hidden_states = hidden_states.index_select(1, token_idx - 1) + else: + hidden_states = hidden_states[:, -1, :] + + if self.config.pretraining_tp > 1: + lm_head_slices = self.lm_head.weight.split(self.vocab_size // self.config.pretraining_tp, dim=0) + logits = [F.linear(hidden_states, lm_head_slices[i]) \ + for i in range(self.config.pretraining_tp)] # pylint: disable=E1102 + logits = torch.cat(logits, dim=-1) + else: + logits = self.lm_head(hidden_states) + logits = logits.float() + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = torch.nn.CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, token_idx=None, **kwargs + ): + past_length = 0 + if not hasattr(self, "kv_past_token_length"): + self.kv_past_token_length = 0 + using_attention_sinks = (hasattr(self, "attention_sink_size") and + hasattr(self, "attention_sink_window_size")) + + reuse_cache = kwargs.get("reuse_cache") + if past_key_values is not None: + if token_idx is not None: + input_ids = torch.index_select(input_ids, 1, token_idx - 1) + else: + if isinstance(past_key_values, Cache): + cache_length = past_key_values.get_seq_length() + past_length = past_key_values.seen_tokens + max_cache_length = past_key_values.get_max_length() + else: + cache_length = past_length = past_key_values[0][0].shape[2] + max_cache_length = None + + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, + # then we are in a setting where + # some of the inputs are exclusively passed as part of the cache + # (e.g. when passing input_embeds as input) + if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: + input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] + # 2 - If the past_length is smaller than input_ids', + # then input_ids holds all input tokens. We can discard input_ids based on the past_length. + elif past_length < input_ids.shape[1]: + input_ids = input_ids[:, past_length:] + # 3 - Otherwise (past_length >= input_ids.shape[1]), + # let's assume input_ids only has unprocessed tokens. + + # If we are about to go beyond the maximum cache length, + # we need to crop the input attention mask. + if ( + max_cache_length is not None + and attention_mask is not None + and cache_length + input_ids.shape[1] > max_cache_length + ): + attention_mask = attention_mask[:, -max_cache_length:] # pylint: disable=E1130 + elif reuse_cache and token_idx is not None: + # With reuse_cache, KV cache is pre allocated hence for the 1st token + # we can slice the inputs till token idx for the fwd pass + input_ids = input_ids[:, :token_idx] + attention_mask = attention_mask[:, :token_idx] + + # prepare postion_ids and attention_mask for attention_sinks + cache_prune_num = 0 + kv_cache_len = kwargs.get("kv_cache_len", None) + position_ids = kwargs.get("position_ids", None) + q_len = input_ids.shape[-1] + if using_attention_sinks: + assert (kv_cache_len and kv_cache_len == self.attention_sink_size + self.attention_sink_window_size) + self.kv_past_token_length = min(self.kv_past_token_length, kv_cache_len) + position_ids = torch.arange(self.kv_past_token_length, + self.kv_past_token_length + q_len, + device=input_ids.device) + attn_sink_mask = torch.ones((q_len, kv_cache_len), device=input_ids.device) + if self.kv_past_token_length < kv_cache_len: + attn_sink_mask[:, self.kv_past_token_length:] = 0 + mask = torch.zeros((q_len, q_len), device=input_ids.device) + mask_cond = torch.arange(mask.size(-1), device=mask.device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 1) + if self.kv_past_token_length + q_len > kv_cache_len: + cache_prune_num = (self.kv_past_token_length + q_len) - kv_cache_len + position_ids = position_ids - cache_prune_num + attn_sink_mask.index_copy_(-1, position_ids, mask) + attention_mask= attn_sink_mask[None, None, :, :].expand(input_ids.shape[0], 1, q_len, kv_cache_len) + position_ids = position_ids.unsqueeze(0) + self.kv_past_token_length += q_len + + if attention_mask is not None and position_ids is None and not using_attention_sinks: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + if token_idx is not None: + position_ids = torch.index_select(position_ids, 1, token_idx - 1) + else: + position_ids = position_ids[:, -input_ids.shape[1] :] + # TODO: we are using token_idx, disable this for now + # if self.generation_config.cache_implementation == "static": + # generation with static cache + # cache_position = kwargs.get("cache_position", None) + # if cache_position is None: + # past_length = 0 + # else: + # past_length = cache_position[-1] + 1 + # input_ids = input_ids[:, past_length:] + # position_ids = position_ids[:, past_length:] + + # TODO @gante we should only keep a `cache_position` in generate, and do +=1. + # same goes for position ids. Could also help with continued generation. + # cache_position = torch.arange(past_length, past_length + position_ids.shape[-1], device=position_ids.device) + # keep cache_position implementation as None for HPU + cache_position = None + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + # The `contiguous()` here is necessary to have a static stride during decoding. torchdynamo otherwise + # TODO: use `next_tokens` directly instead. + model_inputs = {"input_ids": input_ids.contiguous()} + + model_inputs.update( + { + "position_ids": position_ids.contiguous(), + "cache_position": cache_position, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + "token_idx": token_idx, + "trim_logits": kwargs.get("trim_logits"), + "attn_softmax_bf16": kwargs.get("attn_softmax_bf16"), + "reuse_cache": reuse_cache, + "use_flash_attention": kwargs.get("use_flash_attention"), + "flash_attention_recompute": kwargs.get("flash_attention_recompute"), + "flash_attention_causal_mask": kwargs.get("flash_attention_causal_mask"), + "cache_idx": kwargs.get("cache_idx"), + "lazy_mode": kwargs.get("lazy_mode"), + "cache_prune_num": cache_prune_num, + } + ) + return model_inputs + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + # Separate Attention Sink kwargs from regular kwargs + attention_sink_kwargs = {key: value for key, value in kwargs.items() if key.startswith("attention_sink")} + for key in attention_sink_kwargs: + v = kwargs.pop(key) + assert isinstance(v, int) + + model = super().from_pretrained( + pretrained_model_name_or_path, + *model_args, + **kwargs, + ) + + if len(attention_sink_kwargs) > 0: + from intel_extension_for_transformers.transformers.modeling.modeling_gaudi.streaming_llm \ + import enable_streaming_llm + + enable_streaming_llm(model, **attention_sink_kwargs) + model.attention_sink_size = attention_sink_kwargs.get("attention_sink_size") + model.attention_sink_window_size = attention_sink_kwargs.get("attention_sink_window_size") + model.model.attention_sink_size = attention_sink_kwargs.get("attention_sink_size") + model.model.attention_sink_window_size = attention_sink_kwargs.get("attention_sink_window_size") + + return model + + +def apply_customized_rope(q, k, cos, sin, position_ids): + if q.device.type == "hpu" and has_fused_rope: + # TODO: remove `.clone()` when it is fixed in SynapseAI + return FusedRoPE.apply( + q, cos.unsqueeze(0).unsqueeze(0).clone(), sin.unsqueeze(0).unsqueeze(0).clone(), position_ids + ), FusedRoPE.apply( + k, cos.unsqueeze(0).unsqueeze(0).clone(), sin.unsqueeze(0).unsqueeze(0).clone(), position_ids + ) + else: + + return apply_rotary_pos_emb(q, k, cos[position_ids], sin[position_ids]) # pylint: disable=E1120 diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py deleted file mode 100644 index 9e5fa4eb6ca..00000000000 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gpt_neox.py +++ /dev/null @@ -1,682 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import List, Optional, Tuple, Union - -import torch -import torch.nn as nn -from torch.nn import CrossEntropyLoss -from transformers.utils import logging -from transformers.models.gpt_neox.modeling_gpt_neox import ( - _get_unpad_data, - GPTNeoXPreTrainedModel, - GPTNeoXModel, - CausalLMOutputWithPast, - GPT_NEOX_INPUTS_DOCSTRING, - GPT_NEOX_START_DOCSTRING, - _CONFIG_FOR_DOC - -) -from transformers.file_utils import ( - add_start_docstrings, - replace_return_docstrings, - add_start_docstrings_to_model_forward, - ) - -from ..h2o import H2OKVCache, H2OConfig - -logger = logging.get_logger(__name__) - -from packaging import version -import transformers -if version.parse(transformers.__version__) > version.parse("4.33.0"): - from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available - if is_flash_attn_2_available(): - from flash_attn import ( - flash_attn_func, - flash_attn_varlen_func) # pylint: disable=E1101 - from flash_attn.bert_padding import ( - index_first_axis, - pad_input, - unpad_input) # pylint: disable=E1101 - -class H2OGPTNeoXAttention(nn.Module): - def __init__( - self, - config, - h2o_config: H2OConfig, - ): - self.config = config - self.num_attention_heads = config.num_attention_heads - self.hidden_size = config.hidden_size - if self.hidden_size % self.num_attention_heads != 0: - raise ValueError( - "The hidden size is not divisible by the number of attention heads! Make sure to update them" - ) - self.head_size = self.hidden_size // self.num_attention_heads - self.rotary_ndims = int(self.head_size * config.rotary_pct) - self._init_bias(config.max_position_embeddings) - - self.register_buffer("masked_bias", torch.tensor(-1e9), persistent=False) - self._init_rope() - - self.norm_factor = self.head_size**-0.5 - self.query_key_value = nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=config.attention_bias) - self.dense = nn.Linear(config.hidden_size, config.hidden_size, bias=config.attention_bias) - self.attention_dropout = nn.Dropout(config.attention_dropout) - self.is_causal = True - - # for h2o - if real_drop: - real_drop = False - logger.warning_once("GPTNeoXAttention not support for kv cache, usning simulation mode.") - self.h2o_config = h2o_config - self.is_gen = False - self.mean = h2o_config.mean - self.local = h2o_config.local - - self.heavy_ratio = h2o_config.heavy_ratio - self.recent_ratio = h2o_config.recent_ratio - self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) - - def forward( - self, - hidden_states: torch.FloatTensor, - attention_mask: torch.FloatTensor, - position_ids: torch.LongTensor, - head_mask: Optional[torch.FloatTensor] = None, - layer_past: Optional[Tuple[torch.Tensor]] = None, - use_cache: Optional[bool] = False, - output_attentions: Optional[bool] = False, - padding_mask: Optional[torch.Tensor] = None, - ): - has_layer_past = layer_past is not None - - # Compute QKV - # Attention heads [batch, seq_len, hidden_size] - # --> [batch, seq_len, (np * 3 * head_size)] - qkv = self.query_key_value(hidden_states) - - # [batch, seq_len, (num_heads * 3 * head_size)] - # --> [batch, seq_len, num_heads, 3 * head_size] - new_qkv_shape = qkv.size()[:-1] + (self.num_attention_heads, 3 * self.head_size) - qkv = qkv.view(*new_qkv_shape) - - # [batch, seq_len, num_attention_heads, 3 * head_size] --> 3 [batch, num_attention_heads, seq_len, head_size] - query = qkv[..., : self.head_size].permute(0, 2, 1, 3) - key = qkv[..., self.head_size : 2 * self.head_size].permute(0, 2, 1, 3) - value = qkv[..., 2 * self.head_size :].permute(0, 2, 1, 3) - - # Compute rotary embeddings on rotary_ndims - query_rot = query[..., : self.rotary_ndims] - query_pass = query[..., self.rotary_ndims :] - key_rot = key[..., : self.rotary_ndims] - key_pass = key[..., self.rotary_ndims :] - - # Compute token offset for rotary embeddings (when decoding) - seq_len = key.shape[-2] - if has_layer_past: - seq_len += layer_past[0].shape[-2] - cos, sin = self.rotary_emb(value, seq_len=seq_len) - query, key = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids) - query = torch.cat((query, query_pass), dim=-1) - key = torch.cat((key, key_pass), dim=-1) - - # Cache QKV values - if has_layer_past: - past_key = layer_past[0] - past_value = layer_past[1] - key = torch.cat((past_key, key), dim=-2) - value = torch.cat((past_value, value), dim=-2) - present = (key, value) if use_cache else None - - # Compute attention - attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask) - - # Reshape outputs - attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_size) - attn_output = self.dense(attn_output) - - outputs = (attn_output, present) - if output_attentions: - outputs += (attn_weights,) - - return outputs - - @classmethod - def _split_heads(cls, tensor, num_attention_heads, attn_head_size): - """Splits hidden dim into attn_head_size and num_attention_heads.""" - # tensor: [bs, seq_len, hidden_size] - new_shape = tensor.size()[:-1] + (num_attention_heads, attn_head_size) - # -> [bs, seq_len, num_attention_heads, attn_head_size] - tensor = tensor.view(new_shape) - # -> [bs, num_attention_heads, seq_len, attn_head_size] - tensor = tensor.permute(0, 2, 1, 3) - return tensor - - @classmethod - def _merge_heads(cls, tensor, num_attention_heads, attn_head_size): - """Merges attn_head_size dim and num_attn_heads dim into hidden dim.""" - # tensor [bs, num_attention_heads, seq_len, attn_head_size] - tensor = tensor.permute(0, 2, 1, 3).contiguous() - # -> [bs, seq_len, num_attention_heads, attn_head_size] - tensor = tensor.view(tensor.size(0), tensor.size(1), num_attention_heads * attn_head_size) - # -> [bs, seq_len, hidden_size] - return tensor - - def _attn(self, query, key, value, attention_mask=None, head_mask=None): - # q, k, v: [bs, num_attention_heads, seq_len, attn_head_size] - # compute causal mask from causal mask buffer - batch_size, num_attention_heads, query_length, attn_head_size = query.size() - key_length = key.size(-2) - - # dynamically increase the causal mask with the key length, if needed. - if key_length > self.bias.shape[-1]: - self._init_bias(key_length, device=key.device) - causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length] - - query = query.view(batch_size * num_attention_heads, query_length, attn_head_size) - key = key.view(batch_size * num_attention_heads, key_length, attn_head_size) - attn_scores = torch.zeros( - batch_size * num_attention_heads, - query_length, - key_length, - dtype=query.dtype, - device=key.device, - ) - attn_scores = torch.baddbmm( - attn_scores, - query, - key.transpose(1, 2), - beta=1.0, - alpha=self.norm_factor, - ) - attn_scores = attn_scores.view(batch_size, num_attention_heads, query_length, key_length) - - mask_value = torch.finfo(attn_scores.dtype).min - # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`. - # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device` - mask_value = torch.tensor(mask_value, dtype=attn_scores.dtype).to(attn_scores.device) - attn_scores = torch.where(causal_mask, attn_scores, mask_value) - - if attention_mask is not None: - # Apply the attention mask - attn_scores = attn_scores + attention_mask - - attn_weights = nn.functional.softmax(attn_scores, dim=-1) - attn_weights = attn_weights.to(value.dtype) - - # h2o - from ..h2o import get_hh_mask - mask = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights.detach().clone(), local=self.local) - attn_weights[~mask] = torch.finfo(attn_weights.dtype).min - - # Mask heads if we want to - if head_mask is not None: - attn_weights = attn_weights * head_mask - - attn_weights = self.attention_dropout(attn_weights) - - attn_output = torch.matmul(attn_weights, value) - return attn_output, attn_weights - -class H2OGPTNeoXFlashAttention2(H2OGPTNeoXAttention): - """GPTNeoX flash attention module. - - This module inherits from `GPTNeoXAttention` as the weights of the module stays - untouched. The only required change would be on the forward pass where it needs to correctly call the public API of - flash attention and deal with padding tokens in case the input contains any of them. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() - - def forward( - self, - hidden_states: torch.FloatTensor, - attention_mask: torch.FloatTensor, - position_ids: torch.LongTensor, - head_mask: Optional[torch.FloatTensor] = None, - layer_past: Optional[Tuple[torch.Tensor]] = None, - use_cache: Optional[bool] = False, - output_attentions: Optional[bool] = False, - ): - has_layer_past = layer_past is not None - - # Compute QKV - # Attention heads [batch, seq_len, hidden_size] - # --> [batch, seq_len, (np * 3 * head_size)] - qkv = self.query_key_value(hidden_states) - - # [batch, seq_len, (num_heads * 3 * head_size)] - # --> [batch, seq_len, num_heads, 3 * head_size] - new_qkv_shape = qkv.size()[:-1] + (self.num_attention_heads, 3 * self.head_size) - qkv = qkv.view(*new_qkv_shape) - - # [batch, seq_len, num_attention_heads, 3 * head_size] --> 3 [batch, num_attention_heads, seq_len, head_size] - query = qkv[..., : self.head_size].permute(0, 2, 1, 3) - key = qkv[..., self.head_size : 2 * self.head_size].permute(0, 2, 1, 3) - value = qkv[..., 2 * self.head_size :].permute(0, 2, 1, 3) - - query_length = query.shape[-2] - - # Compute rotary embeddings on rotary_ndims - query_rot = query[..., : self.rotary_ndims] - query_pass = query[..., self.rotary_ndims :] - key_rot = key[..., : self.rotary_ndims] - key_pass = key[..., self.rotary_ndims :] - - # Compute token offset for rotary embeddings (when decoding) - seq_len = key.shape[-2] - if has_layer_past: - seq_len += layer_past[0].shape[-2] - cos, sin = self.rotary_emb(value, seq_len=seq_len) - query, key = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids) - query = torch.cat((query, query_pass), dim=-1) - key = torch.cat((key, key_pass), dim=-1) - - # Cache QKV values - if has_layer_past: - past_key = layer_past[0] - past_value = layer_past[1] - key = torch.cat((past_key, key), dim=-2) - value = torch.cat((past_value, value), dim=-2) - present = (key, value) if use_cache else None - - # GPT-neo-X casts query and key in fp32 to apply rotary embedding in full precision - target_dtype = value.dtype - if query.dtype != target_dtype: - query = query.to(target_dtype) - if key.dtype != target_dtype: - key = key.to(target_dtype) - - # Permute to get the expected shape for Flash Attention - query = query.permute(0, 2, 1, 3) - key = key.permute(0, 2, 1, 3) - value = value.permute(0, 2, 1, 3) - - # In PEFT, usually we cast the layer norms in float32 for training stability reasons - # therefore the input hidden states gets silently casted in float32. Hence, we need - # cast them back in float16 / bfloat16 just to be sure everything works as expected. - # This might slowdown training & inference so it is recommended to not cast the LayerNorms - input_dtype = query.dtype - if input_dtype == torch.float32: - if torch.is_autocast_enabled(): - target_dtype = torch.get_autocast_gpu_dtype() - # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype - else: - target_dtype = self.query_key_value.weight.dtype - - logger.warning_once( - f"The input hidden states seems to be silently casted in float32, this might be related to" - f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" - f" {target_dtype}." - ) - - query = query.to(target_dtype) - key = key.to(target_dtype) - value = value.to(target_dtype) - - attention_dropout = self.config.attention_dropout if self.training else 0.0 - - # Compute attention - attn_weights = self._flash_attention_forward( - query, key, value, attention_mask, query_length, dropout=attention_dropout, softmax_scale=self.norm_factor - ) - - # h2o - from ..h2o import get_hh_mask - mask = get_hh_mask(self.heavy_ratio, self.recent_ratio, attn_weights.detach().clone(), local=self.local) - attn_weights[~mask] = torch.finfo(attn_weights.dtype).min - - # Reshape outputs - attn_output = attn_weights.reshape( - attn_weights.shape[0], attn_weights.shape[1], self.num_attention_heads * self.head_size - ) - attn_output = self.dense(attn_output) - - outputs = (attn_output, present) - if output_attentions: - outputs += (attn_weights,) - - return outputs - - # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._flash_attention_forward - def _flash_attention_forward( - self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None - ): - """ - Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token - first unpad the input, then computes the attention scores and pad the final attention scores. - - Args: - query_states (`torch.Tensor`): - Input query states to be passed to Flash Attention API - key_states (`torch.Tensor`): - Input key states to be passed to Flash Attention API - value_states (`torch.Tensor`): - Input value states to be passed to Flash Attention API - attention_mask (`torch.Tensor`): - The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the - position of padding tokens and 1 for the position of non-padding tokens. - dropout (`float`): - Attention dropout - softmax_scale (`float`, *optional*): - The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) - """ - if not self._flash_attn_uses_top_left_mask: - causal = self.is_causal - else: - # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. - causal = self.is_causal and query_length != 1 - - # Contains at least one padding token in the sequence - if attention_mask is not None: - batch_size = query_states.shape[0] - query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( - query_states, key_states, value_states, attention_mask, query_length - ) - - cu_seqlens_q, cu_seqlens_k = cu_seq_lens - max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens - - attn_output_unpad = flash_attn_varlen_func( - query_states, - key_states, - value_states, - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_in_batch_q, - max_seqlen_k=max_seqlen_in_batch_k, - dropout_p=dropout, - softmax_scale=softmax_scale, - causal=causal, - ) - - attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) - else: - attn_output = flash_attn_func( - query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal - ) - - return attn_output - - # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._upad_input with num_heads->num_attention_heads - def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): - indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) - batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape - - key_layer = index_first_axis( - key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k - ) - value_layer = index_first_axis( - value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k - ) - if query_length == kv_seq_len: - query_layer = index_first_axis( - query_layer.reshape(batch_size * kv_seq_len, self.num_attention_heads, head_dim), indices_k - ) - cu_seqlens_q = cu_seqlens_k - max_seqlen_in_batch_q = max_seqlen_in_batch_k - indices_q = indices_k - elif query_length == 1: - max_seqlen_in_batch_q = 1 - cu_seqlens_q = torch.arange( - batch_size + 1, dtype=torch.int32, device=query_layer.device - ) # There is a memcpy here, that is very bad. - indices_q = cu_seqlens_q[:-1] - query_layer = query_layer.squeeze(1) - else: - # The -q_len: slice assumes left padding. - attention_mask = attention_mask[:, -query_length:] - query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) - - return ( - query_layer, - key_layer, - value_layer, - indices_q, - (cu_seqlens_q, cu_seqlens_k), - (max_seqlen_in_batch_q, max_seqlen_in_batch_k), - ) - -def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): - """Applies Rotary Position Embedding to the query and key tensors. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - position_ids (`torch.Tensor`): - The position indices of the tokens corresponding to the query and key tensors. For example, this can be - used to pass offsetted position ids when working with a KV-cache. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - cos = cos[position_ids].unsqueeze(unsqueeze_dim) - sin = sin[position_ids].unsqueeze(unsqueeze_dim) - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -@add_start_docstrings( - """GPTNeoX Model with a `language modeling` head on top for CLM fine-tuning.""", GPT_NEOX_START_DOCSTRING -) -class H2OGPTNeoXForCausalLM(GPTNeoXPreTrainedModel): - _tied_weights_keys = ["embed_out.weight"] - - def __init__(self, config, h2o_config): - super().__init__(config) - - self.gpt_neox = GPTNeoXModel(config) - self.embed_out = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - - num_layers = len(self.model.layers) - for layer_idx in range(num_layers): - module = self.gpt_neox.layers[layer_idx].self_attn - cls_name = module.__class__.__name__ - if cls_name == "GPTNeoXFlashAttention2": - cls = H2OGPTNeoXFlashAttention2 - else: - cls = H2OGPTNeoXAttention - - self.gpt_neox.layers[layer_idx].self_attn = cls( - config, - layer_idx, - h2o_config - ) - - # Initialize weights and apply final processing - self.post_init() - - def get_output_embeddings(self): - return self.embed_out - - def set_output_embeddings(self, new_embeddings): - self.embed_out = new_embeddings - - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.FloatTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - head_mask: Optional[torch.FloatTensor] = None, - past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, - labels: Optional[torch.LongTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - r"""past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional tensors are - only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks that can be used (see - `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in - `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are - ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: - - Example: - - ```python - >>> from transformers import AutoTokenizer, GPTNeoXForCausalLM, GPTNeoXConfig - >>> import torch - - >>> tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b") - >>> config = GPTNeoXConfig.from_pretrained("EleutherAI/gpt-neox-20b") - >>> config.is_decoder = True - >>> model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/gpt-neox-20b", config=config) - - >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") - >>> outputs = model(**inputs) - - >>> prediction_logits = outputs.logits - ``` - """ - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - outputs = self.gpt_neox( - input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - past_key_values=past_key_values, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - - hidden_states = outputs[0] - lm_logits = self.embed_out(hidden_states) - - lm_loss = None - if labels is not None: - # move labels to correct device to enable model parallelism - labels = labels.to(lm_logits.device) - # we are doing next-token prediction; shift prediction scores and input ids by one - shift_logits = lm_logits[:, :-1, :].contiguous() - labels = labels[:, 1:].contiguous() - loss_fct = CrossEntropyLoss() - lm_loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), labels.view(-1)) - - if not return_dict: - output = (lm_logits,) + outputs[1:] - return ((lm_loss,) + output) if lm_loss is not None else output - - return CausalLMOutputWithPast( - loss=lm_loss, - logits=lm_logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) - - def prepare_inputs_for_generation( - self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs - ): - input_shape = input_ids.shape - # cut decoder_input_ids if past is used - if past_key_values is not None: - past_length = past_key_values[0][0].shape[2] - - # Some generation methods already pass only the last input ID - if input_ids.shape[1] > past_length: - remove_prefix_length = past_length - else: - # Default to old behavior: keep only final ID - remove_prefix_length = input_ids.shape[1] - 1 - - input_ids = input_ids[:, remove_prefix_length:] - - position_ids = kwargs.get("position_ids", None) - if attention_mask is not None and position_ids is None: - # create position_ids on the fly for batch generation - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - if past_key_values: - position_ids = position_ids[:, -input_ids.shape[1] :] - - # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly - if attention_mask is None: - attention_mask = input_ids.new_ones(input_shape) - - # if `inputs_embeds` are passed, we only want to use them in the 1st generation step - if inputs_embeds is not None and past_key_values is None: - model_inputs = {"inputs_embeds": inputs_embeds} - else: - model_inputs = {"input_ids": input_ids} - model_inputs.update( - { - "attention_mask": attention_mask, - "past_key_values": past_key_values, - "position_ids": position_ids, - "use_cache": kwargs.get("use_cache"), - } - ) - - return model_inputs - - def _reorder_cache(self, past_key_values, beam_idx): - reordered_past = () - for layer_past in past_key_values: - reordered_past += ( - tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past[:2]) - + layer_past[2:], - ) - return reordered_past diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py deleted file mode 100644 index 2451f94d190..00000000000 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mistral.py +++ /dev/null @@ -1,880 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import math -import inspect -import logging -from typing import List, Optional, Tuple, Union -from functools import partial - -import torch -import torch.nn as nn -from torch.nn import CrossEntropyLoss - -from transformers.cache_utils import Cache -from transformers.models.mistral.modeling_mistral import ( - apply_rotary_pos_emb, - repeat_kv, - _get_unpad_data, - MistralConfig, - MistralModel, - MistralPreTrainedModel, - MistralRotaryEmbedding, - CausalLMOutputWithPast, - MISTRAL_INPUTS_DOCSTRING, - _CONFIG_FOR_DOC, - - ) -from transformers.file_utils import ( - replace_return_docstrings, - add_start_docstrings_to_model_forward, - ) - -from ..h2o import H2OKVCache, H2OConfig, generate - -logger = logging.getLogger(__name__) - -from packaging import version -import transformers -if version.parse(transformers.__version__) > version.parse("4.33.0"): - from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available - if is_flash_attn_2_available(): - from flash_attn import ( - flash_attn_func, - flash_attn_varlen_func) # pylint: disable=E1101 - from flash_attn.bert_padding import ( - index_first_axis, - pad_input, - unpad_input) # pylint: disable=E1101 - _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) - -class H2OMistralAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper. - - Modified to use sliding window attention: Longformer - and "Generating Long Sequences with Sparse Transformers". - """ - - def __init__( - self, config: MistralConfig, - layer_idx: Optional[int] = None, - h2o_config: H2OConfig = None, - ): - super().__init__() - self.config = config - self.layer_idx = layer_idx - if layer_idx is None: - logger.warning_once( - f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " - "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " - "when creating this class." - ) - - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.max_position_embeddings = config.max_position_embeddings - self.rope_theta = config.rope_theta - self.is_causal = True - self.attention_dropout = config.attention_dropout - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError( - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads})." - ) - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) - self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) - - self.rotary_emb = MistralRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - base=self.rope_theta, - ) - - # for h2o - self.h2o_config = h2o_config - self.is_gen = False - self.real_drop = h2o_config.real_drop - self.mean = h2o_config.mean - self.local = h2o_config.local - - self.heavy_ratio = h2o_config.heavy_ratio - self.recent_ratio = h2o_config.recent_ratio - self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - **kwargs, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - if "padding_mask" in kwargs: - logger.warn( - "Passing `padding_mask` is deprecated and will be removed in v4.37. " - "Please make sure use `attention_mask` instead.`" - ) - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - if self.layer_idx is None: - raise ValueError( - f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " - "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " - "with a layer index." - ) - kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) - cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - - if past_key_value is not None: - cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - # repeat k/v heads if n_kv_heads < n_heads - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}" - ) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - - attn_weights = attn_weights + attention_mask - - # H2O - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop and past_key_value is not None: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights.detach().clone(), - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - self.h2o_kv_cache.past_length += attn_weights.size(-2) - else: - from ..h2o import get_hh_mask - mask = get_hh_mask( - self.heavy_ratio, - self.recent_ratio, - attn_weights.detach().clone(), - local=self.local) - attn_weights[~mask] = torch.finfo(attn_weights.dtype).min - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) - - attn_output = torch.matmul(attn_weights, value_states) - - if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) - - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - -class H2OMistralFlashAttention2(H2OMistralAttention): - """Mistral flash attention module. - - This module inherits from `MistralAttention` as the weights of the module stays - untouched. The only required change would be on the forward pass where it needs to correctly call the public API of - flash attention and deal with padding tokens in case the input contains any of them. - """ - - # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - ): - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - if self.layer_idx is None: - raise ValueError( - f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " - "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " - "with a layer index." - ) - kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) - - # Because the input can be padded, the absolute sequence length depends on the max position id. - rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1 - cos, sin = self.rotary_emb(value_states, seq_len=rotary_seq_len) - - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - - use_sliding_windows = ( - _flash_supports_window_size - and getattr(self.config, "sliding_window", None) is not None - and kv_seq_len > self.config.sliding_window - ) - - if not _flash_supports_window_size: - logger.warning_once( - "The current flash attention version does not support sliding window attention, for a more memory efficient implementation" - " make sure to upgrade flash-attn library." - ) - - if past_key_value is not None: - # Activate slicing cache only if the config has a value `sliding_windows` attribute - cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 - if ( - getattr(self.config, "sliding_window", None) is not None - and kv_seq_len > self.config.sliding_window - and cache_has_contents - ): - slicing_tokens = 1 - self.config.sliding_window - - past_key = past_key_value[self.layer_idx][0] - past_value = past_key_value[self.layer_idx][1] - - past_key = past_key[:, :, slicing_tokens:, :].contiguous() - past_value = past_value[:, :, slicing_tokens:, :].contiguous() - - if past_key.shape[-2] != self.config.sliding_window - 1: - raise ValueError( - f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" - f" {past_key.shape}" - ) - - if attention_mask is not None: - attention_mask = attention_mask[:, slicing_tokens:] - attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) - - cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - # repeat k/v heads if n_kv_heads < n_heads - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - dropout_rate = 0.0 if not self.training else self.attention_dropout - - # In PEFT, usually we cast the layer norms in float32 for training stability reasons - # therefore the input hidden states gets silently casted in float32. Hence, we need - # cast them back in float16 just to be sure everything works as expected. - input_dtype = query_states.dtype - if input_dtype == torch.float32: - if torch.is_autocast_enabled(): - target_dtype = torch.get_autocast_gpu_dtype() - # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype - else: - target_dtype = self.q_proj.weight.dtype - - logger.warning_once( - f"The input hidden states seems to be silently casted in float32, this might be related to" - f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" - f" {target_dtype}." - ) - - query_states = query_states.to(target_dtype) - key_states = key_states.to(target_dtype) - value_states = value_states.to(target_dtype) - - # Reashape to the expected shape for Flash Attention - query_states = query_states.transpose(1, 2) - key_states = key_states.transpose(1, 2) - value_states = value_states.transpose(1, 2) - - # h2o - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}" - ) - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights + attention_mask - - if past_key_value is not None: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - self.h2o_kv_cache.past_length += attn_weights.size(-2) - - attn_output = self._flash_attention_forward( - query_states, - key_states, - value_states, - attention_mask, - q_len, - dropout=dropout_rate, - use_sliding_windows=use_sliding_windows, - ) - - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - - def _flash_attention_forward( - self, - query_states, - key_states, - value_states, - attention_mask, - query_length, - dropout=0.0, - softmax_scale=None, - use_sliding_windows=False, - ): - """ - Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token - first unpad the input, then computes the attention scores and pad the final attention scores. - - Args: - query_states (`torch.Tensor`): - Input query states to be passed to Flash Attention API - key_states (`torch.Tensor`): - Input key states to be passed to Flash Attention API - value_states (`torch.Tensor`): - Input value states to be passed to Flash Attention API - attention_mask (`torch.Tensor`): - The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the - position of padding tokens and 1 for the position of non-padding tokens. - dropout (`float`): - Attention dropout - softmax_scale (`float`, *optional*): - The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) - use_sliding_windows (`bool`, *optional*): - Whether to activate sliding window attention. - """ - if not self._flash_attn_uses_top_left_mask: - causal = self.is_causal - else: - # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. - causal = self.is_causal and query_length != 1 - - # Contains at least one padding token in the sequence - if attention_mask is not None: - batch_size = query_states.shape[0] - query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( - query_states, key_states, value_states, attention_mask, query_length - ) - - cu_seqlens_q, cu_seqlens_k = cu_seq_lens - max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens - - if not use_sliding_windows: - attn_output_unpad = flash_attn_varlen_func( - query_states, - key_states, - value_states, - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_in_batch_q, - max_seqlen_k=max_seqlen_in_batch_k, - dropout_p=dropout, - softmax_scale=softmax_scale, - causal=causal, - ) - else: - attn_output_unpad = flash_attn_varlen_func( - query_states, - key_states, - value_states, - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_in_batch_q, - max_seqlen_k=max_seqlen_in_batch_k, - dropout_p=dropout, - softmax_scale=softmax_scale, - causal=causal, - window_size=(self.config.sliding_window, self.config.sliding_window), - ) - - attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) - else: - if not use_sliding_windows: - attn_output = flash_attn_func( - query_states, - key_states, - value_states, - dropout, - softmax_scale=softmax_scale, - causal=causal, - ) - else: - attn_output = flash_attn_func( - query_states, - key_states, - value_states, - dropout, - softmax_scale=softmax_scale, - causal=causal, - window_size=(self.config.sliding_window, self.config.sliding_window), - ) - - return attn_output - - def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): - batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape - - # On the first iteration we need to properly re-create the padding mask - # by slicing it on the proper place - if kv_seq_len != attention_mask.shape[-1]: - attention_mask_num_tokens = attention_mask.shape[-1] - attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :] - - indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) - - key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) - value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) - - if query_length == kv_seq_len: - query_layer = index_first_axis( - query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k - ) - cu_seqlens_q = cu_seqlens_k - max_seqlen_in_batch_q = max_seqlen_in_batch_k - indices_q = indices_k - elif query_length == 1: - max_seqlen_in_batch_q = 1 - cu_seqlens_q = torch.arange( - batch_size + 1, dtype=torch.int32, device=query_layer.device - ) # There is a memcpy here, that is very bad. - indices_q = cu_seqlens_q[:-1] - query_layer = query_layer.squeeze(1) - else: - # The -q_len: slice assumes left padding. - attention_mask = attention_mask[:, -query_length:] - query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) - - return ( - query_layer, - key_layer, - value_layer, - indices_q, - (cu_seqlens_q, cu_seqlens_k), - (max_seqlen_in_batch_q, max_seqlen_in_batch_k), - ) - -class H2OMistralSdpaAttention(H2OMistralAttention): - """Mistral attention module using torch.nn.functional.scaled_dot_product_attention. - - This module inherits from `MistralAttention` as the weights of the module stays untouched. - The only changes are on the forward pass to adapt to SDPA API. - """ - - # Adapted from MistralAttention.forward - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - if output_attentions: - # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. - logger.warning_once( - "MistralModel is using MistralSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention`" - "does not support `output_attentions=True`. Falling back to the manual attention implementation, " - 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards.' - ' This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' - ) - return super().forward( - hidden_states=hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_value=past_key_value, - output_attentions=output_attentions, - use_cache=use_cache, - ) - - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) - cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) - - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - - if past_key_value is not None: - cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - - # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged - # with non-contiguous inputs with custom attn_mask, - # Reference: https://github.com/pytorch/pytorch/issues/112577. - - - # h2o - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}" - ) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, \ - but is {attention_mask.size()}" - ) - - attn_weights = attn_weights + attention_mask - - if past_key_value is not None: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - self.h2o_kv_cache.past_length += attn_weights.size(-2) - - if query_states.device.type == "cuda" and attention_mask is not None: - query_states = query_states.contiguous() - key_states = key_states.contiguous() - value_states = value_states.contiguous() - - attn_output = torch.nn.functional.scaled_dot_product_attention( - query_states, - key_states, - value_states, - attn_mask=attention_mask, - dropout_p=self.attention_dropout if self.training else 0.0, - # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d - # that does not create a causal mask in case q_len == 1. - is_causal=self.is_causal and attention_mask is None and q_len > 1, - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.view(bsz, q_len, self.hidden_size) - - attn_output = self.o_proj(attn_output) - - return attn_output, None, past_key_value - -class H2OMistralForCausalLM(MistralPreTrainedModel): - _tied_weights_keys = ["lm_head.weight"] - - def __init__( - self, - config, - h2o_config: H2OConfig): - super().__init__(config) - self.model = MistralModel(config) - self.vocab_size = config.vocab_size - self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - - num_layers = len(self.model.layers) - for layer_idx in range(num_layers): - module = self.model.layers[layer_idx].self_attn - cls_name = module.__class__.__name__ - if not h2o_config.real_drop: - cls = H2OMistralAttention - elif cls_name == "MistralFlashAttention2": - cls = H2OMistralFlashAttention2 - elif cls_name == "LlamaSdpaAttention": - cls = H2OMistralSdpaAttention - else: - cls = H2OMistralAttention - - self.model.layers[layer_idx].self_attn = cls( - config, - layer_idx, - h2o_config - ) - - # Initialize weights and apply final processing - self.post_init() - - self.ori_generate = self.generate - self.generate = partial(generate, self) - - def get_input_embeddings(self): - return self.model.embed_tokens - - def set_input_embeddings(self, value): - self.model.embed_tokens = value - - def get_output_embeddings(self): - return self.lm_head - - def set_output_embeddings(self, new_embeddings): - self.lm_head = new_embeddings - - def set_decoder(self, decoder): - self.model = decoder - - def get_decoder(self): - return self.model - - @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - - Example: - - ```python - >>> from transformers import AutoTokenizer, MistralForCausalLM - - >>> model = MistralForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1") - >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1") - - >>> prompt = "Hey, are you conscious? Can you talk to me?" - >>> inputs = tokenizer(prompt, return_tensors="pt") - - >>> # Generate - >>> generate_ids = model.generate(inputs.input_ids, max_length=30) - >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." - ```""" - - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) - outputs = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - - hidden_states = outputs[0] - logits = self.lm_head(hidden_states) - logits = logits.float() - - loss = None - if labels is not None: - # Shift so that tokens < n predict n - shift_logits = logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - # Flatten the tokens - shift_logits = shift_logits.view(-1, self.config.vocab_size) - shift_labels = shift_labels.view(-1) - # Ensure tensors are on the same device - shift_labels = shift_labels.to(shift_logits.device) - loss_fct = CrossEntropyLoss() - loss = loss_fct(shift_logits, shift_labels) - - if not return_dict: - output = (logits,) + outputs[1:] - return (loss,) + output if loss is not None else output - - return CausalLMOutputWithPast( - loss=loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) - - def prepare_inputs_for_generation( - self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs - ): - # Omit tokens covered by past_key_values - if past_key_values is not None: - if isinstance(past_key_values, Cache): - cache_length = past_key_values.get_seq_length() - past_length = past_key_values.seen_tokens - max_cache_length = past_key_values.get_max_length() - else: - cache_length = past_length = past_key_values[0][0].shape[2] - max_cache_length = None - - cache_length = past_length = self.model.layers[0].self_attn.h2o_kv_cache.past_length - max_cache_length = None - - # Keep only the unprocessed tokens: - # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where - # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as - # input) - if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: - input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] - # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard - # input_ids based on the past_length. - elif past_length < input_ids.shape[1]: - input_ids = input_ids[:, past_length:] - # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. - - # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. - if ( - max_cache_length is not None - and attention_mask is not None - and cache_length + input_ids.shape[1] > max_cache_length - ): - attention_mask = attention_mask[:, -max_cache_length:] - - position_ids = kwargs.get("position_ids", None) - if attention_mask is not None and position_ids is None: - # create position_ids on the fly for batch generation - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - if past_key_values: - position_ids = position_ids[:, -input_ids.shape[1] :] - - # if `inputs_embeds` are passed, we only want to use them in the 1st generation step - if inputs_embeds is not None and past_key_values is None: - model_inputs = {"inputs_embeds": inputs_embeds} - else: - model_inputs = {"input_ids": input_ids} - - model_inputs.update( - { - "position_ids": position_ids, - "past_key_values": past_key_values, - "use_cache": kwargs.get("use_cache"), - "attention_mask": attention_mask, - } - ) - return model_inputs - - @staticmethod - def _reorder_cache(past_key_values, beam_idx): - reordered_past = () - for layer_past in past_key_values: - reordered_past += ( - tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), - ) - return reordered_past diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py deleted file mode 100644 index 168db78b860..00000000000 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_mixtral.py +++ /dev/null @@ -1,895 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import inspect -import math -import logging -from typing import List, Optional, Tuple, Union -from functools import partial - -import torch -import torch.nn as nn -from torch.nn import CrossEntropyLoss - -from transformers.cache_utils import Cache -from transformers.models.mixtral.modeling_mixtral import ( - apply_rotary_pos_emb, - repeat_kv, - _get_unpad_data, - load_balancing_loss_func, - MixtralConfig, - MixtralRotaryEmbedding, - MixtralModel, - MixtralPreTrainedModel, - MoeCausalLMOutputWithPast, - MIXTRAL_INPUTS_DOCSTRING, - _CONFIG_FOR_DOC, - ) -from transformers.file_utils import ( - replace_return_docstrings, - add_start_docstrings_to_model_forward, - ) -from transformers.utils import is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10 - -from ..h2o import H2OKVCache, H2OConfig, generate - -logger = logging.getLogger(__name__) - -if is_flash_attn_2_available(): - from flash_attn import flash_attn_func, flash_attn_varlen_func - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - - _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) - -class H2OMixtralAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper. - - Modified to use sliding window attention: Longformer - and "Generating Long Sequences with Sparse Transformers". - """ - - def __init__( - self, config: MixtralConfig, - layer_idx: Optional[int] = None, - h2o_config: H2OConfig = None, - ): - super().__init__() - self.config = config - self.layer_idx = layer_idx - if layer_idx is None: - logger.warning_once( - f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " - "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " - "when creating this class." - ) - - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.max_position_embeddings = config.max_position_embeddings - self.rope_theta = config.rope_theta - self.is_causal = True - self.attention_dropout = config.attention_dropout - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError( - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads})." - ) - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) - self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) - - self.rotary_emb = MixtralRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - base=self.rope_theta, - ) - - # for h2o - self.h2o_config = h2o_config - self.is_gen = False - self.real_drop = h2o_config.real_drop - self.mean = h2o_config.mean - self.local = h2o_config.local - - self.heavy_ratio = h2o_config.heavy_ratio - self.recent_ratio = h2o_config.recent_ratio - self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - **kwargs, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - if "padding_mask" in kwargs: - logger.warn( - "Passing `padding_mask` is deprecated and will be removed in v4.37." - "Please make sure use `attention_mask` instead.`" - ) - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - if self.layer_idx is None: - raise ValueError( - f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " - "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " - "with a layer index." - ) - kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) - cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - - if past_key_value is not None: - cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - # repeat k/v heads if n_kv_heads < n_heads - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}" - ) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - - attn_weights = attn_weights + attention_mask - - - # H2O - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop and past_key_value is not None: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights.detach().clone(), - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - self.h2o_kv_cache.past_length += attn_weights.size(-2) - else: - from ..h2o import get_hh_mask - mask = get_hh_mask( - self.heavy_ratio, - self.recent_ratio, - attn_weights.detach().clone(), - local=self.local) - attn_weights[~mask] = torch.finfo(attn_weights.dtype).min - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) - attn_output = torch.matmul(attn_weights, value_states) - - if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) - - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - -class H2OMixtralFlashAttention2(H2OMixtralAttention): - """Mixtral flash attention module. - - This module inherits from `MixtralAttention` as the weights of the module stays - untouched. The only required change would be on the forward pass where it needs to correctly call the public API of - flash attention and deal with padding tokens in case the input contains any of them. - """ - - # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, - # that was made default for flash_attn>=2.1. This attribute is used to handle this difference. - # Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - ): - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - if self.layer_idx is None: - raise ValueError( - f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " - "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " - "with a layer index." - ) - kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) - - # Because the input can be padded, the absolute sequence length depends on the max position id. - rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1 - cos, sin = self.rotary_emb(value_states, seq_len=rotary_seq_len) - - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - - use_sliding_windows = ( - _flash_supports_window_size - and getattr(self.config, "sliding_window", None) is not None - and kv_seq_len > self.config.sliding_window - ) - - if not _flash_supports_window_size: - logger.warning_once( - "The current flash attention version does not support sliding window attention," - "for a more memory efficient implementation" - " make sure to upgrade flash-attn library." - ) - - if past_key_value is not None: - # Activate slicing cache only if the config has a value `sliding_windows` attribute - cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 - if ( - getattr(self.config, "sliding_window", None) is not None - and kv_seq_len > self.config.sliding_window - and cache_has_contents - ): - slicing_tokens = 1 - self.config.sliding_window - - past_key = past_key_value[self.layer_idx][0] - past_value = past_key_value[self.layer_idx][1] - - past_key = past_key[:, :, slicing_tokens:, :].contiguous() - past_value = past_value[:, :, slicing_tokens:, :].contiguous() - - if past_key.shape[-2] != self.config.sliding_window - 1: - raise ValueError( - f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" - f" {past_key.shape}" - ) - - if attention_mask is not None: - attention_mask = attention_mask[:, slicing_tokens:] - attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) - - cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - # repeat k/v heads if n_kv_heads < n_heads - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - dropout_rate = 0.0 if not self.training else self.attention_dropout - - # h2o - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}" - ) - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights + attention_mask - if past_key_value is not None: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - self.h2o_kv_cache.past_length += attn_weights.size(-2) - - # In PEFT, usually we cast the layer norms in float32 for training stability reasons - # therefore the input hidden states gets silently casted in float32. Hence, we need - # cast them back in float16 just to be sure everything works as expected. - input_dtype = query_states.dtype - if input_dtype == torch.float32: - if torch.is_autocast_enabled(): - target_dtype = torch.get_autocast_gpu_dtype() - # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype - else: - target_dtype = self.q_proj.weight.dtype - - logger.warning_once( - f"The input hidden states seems to be silently casted in float32, this might be related to" - f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" - f" {target_dtype}." - ) - - query_states = query_states.to(target_dtype) - key_states = key_states.to(target_dtype) - value_states = value_states.to(target_dtype) - - # Reashape to the expected shape for Flash Attention - query_states = query_states.transpose(1, 2) - key_states = key_states.transpose(1, 2) - value_states = value_states.transpose(1, 2) - - attn_output = self._flash_attention_forward( - query_states, - key_states, - value_states, - attention_mask, - q_len, - dropout=dropout_rate, - use_sliding_windows=use_sliding_windows, - ) - - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - - def _flash_attention_forward( - self, - query_states, - key_states, - value_states, - attention_mask, - query_length, - dropout=0.0, - softmax_scale=None, - use_sliding_windows=False, - ): - """ - Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token - first unpad the input, then computes the attention scores and pad the final attention scores. - - Args: - query_states (`torch.Tensor`): - Input query states to be passed to Flash Attention API - key_states (`torch.Tensor`): - Input key states to be passed to Flash Attention API - value_states (`torch.Tensor`): - Input value states to be passed to Flash Attention API - attention_mask (`torch.Tensor`): - The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the - position of padding tokens and 1 for the position of non-padding tokens. - dropout (`float`): - Attention dropout - softmax_scale (`float`, *optional*): - The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) - use_sliding_windows (`bool`, *optional*): - Whether to activate sliding window attention. - """ - if not self._flash_attn_uses_top_left_mask: - causal = self.is_causal - else: - # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. - causal = self.is_causal and query_length != 1 - - # Contains at least one padding token in the sequence - if attention_mask is not None: - batch_size = query_states.shape[0] - query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( - query_states, key_states, value_states, attention_mask, query_length - ) - - cu_seqlens_q, cu_seqlens_k = cu_seq_lens - max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens - - if not use_sliding_windows: - attn_output_unpad = flash_attn_varlen_func( - query_states, - key_states, - value_states, - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_in_batch_q, - max_seqlen_k=max_seqlen_in_batch_k, - dropout_p=dropout, - softmax_scale=softmax_scale, - causal=causal, - ) - else: - attn_output_unpad = flash_attn_varlen_func( - query_states, - key_states, - value_states, - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_in_batch_q, - max_seqlen_k=max_seqlen_in_batch_k, - dropout_p=dropout, - softmax_scale=softmax_scale, - causal=causal, - window_size=(self.config.sliding_window, self.config.sliding_window), - ) - - attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) - else: - if not use_sliding_windows: - attn_output = flash_attn_func( - query_states, - key_states, - value_states, - dropout, - softmax_scale=softmax_scale, - causal=causal, - ) - else: - attn_output = flash_attn_func( - query_states, - key_states, - value_states, - dropout, - softmax_scale=softmax_scale, - causal=causal, - window_size=(self.config.sliding_window, self.config.sliding_window), - ) - - return attn_output - - def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): - batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape - - # On the first iteration we need to properly re-create the padding mask - # by slicing it on the proper place - if kv_seq_len != attention_mask.shape[-1]: - attention_mask_num_tokens = attention_mask.shape[-1] - attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :] - - indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) - - key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) - value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) - - if query_length == kv_seq_len: - query_layer = index_first_axis( - query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k - ) - cu_seqlens_q = cu_seqlens_k - max_seqlen_in_batch_q = max_seqlen_in_batch_k - indices_q = indices_k - elif query_length == 1: - max_seqlen_in_batch_q = 1 - cu_seqlens_q = torch.arange( - batch_size + 1, dtype=torch.int32, device=query_layer.device - ) # There is a memcpy here, that is very bad. - indices_q = cu_seqlens_q[:-1] - query_layer = query_layer.squeeze(1) - else: - # The -q_len: slice assumes left padding. - attention_mask = attention_mask[:, -query_length:] - query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) - - return ( - query_layer, - key_layer, - value_layer, - indices_q, - (cu_seqlens_q, cu_seqlens_k), - (max_seqlen_in_batch_q, max_seqlen_in_batch_k), - ) - -class H2OMixtralSdpaAttention(H2OMixtralAttention): - """Mixtral attention module using torch.nn.functional.scaled_dot_product_attention. - - This module inherits from - `MixtralAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to - SDPA API. - """ - - # Adapted from MixtralAttention.forward - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - if output_attentions: - # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. - logger.warning_once( - "MixtralModel is using MixtralSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention`" - "does not support `output_attentions=True`. Falling back to the manual attention implementation, " - 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards.' - 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' - ) - return super().forward( - hidden_states=hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_value=past_key_value, - output_attentions=output_attentions, - use_cache=use_cache, - ) - - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - kv_seq_len = key_states.shape[-2] - if past_key_value is not None: - kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) - cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) - - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - - if past_key_value is not None: - cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - # h2o - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}" - ) - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights + attention_mask - if past_key_value is not None: - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value.key_cache[self.layer_idx], - past_key_value.value_cache[self.layer_idx], - mean=self.mean - ) - past_key_value.key_cache[self.layer_idx] = new_key_states - past_key_value.value_cache[self.layer_idx] = new_value_states - self.h2o_kv_cache.past_length += attn_weights.size(-2) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" - ) - - # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, - # Reference: https://github.com/pytorch/pytorch/issues/112577. - if query_states.device.type == "cuda" and attention_mask is not None: - query_states = query_states.contiguous() - key_states = key_states.contiguous() - value_states = value_states.contiguous() - - attn_output = torch.nn.functional.scaled_dot_product_attention( - query_states, - key_states, - value_states, - attn_mask=attention_mask, - dropout_p=self.attention_dropout if self.training else 0.0, - # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1. - is_causal=self.is_causal and attention_mask is None and q_len > 1, - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.view(bsz, q_len, self.hidden_size) - - attn_output = self.o_proj(attn_output) - - return attn_output, None, past_key_value - - -class H2OMixtralForCausalLM(MixtralPreTrainedModel): - _tied_weights_keys = ["lm_head.weight"] - - def __init__(self, config, h2o_config: H2OConfig): - super().__init__(config) - self.model = MixtralModel(config) - self.vocab_size = config.vocab_size - self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - self.router_aux_loss_coef = config.router_aux_loss_coef - self.num_experts = config.num_local_experts - self.num_experts_per_tok = config.num_experts_per_tok - - num_layers = len(self.model.layers) - for layer_idx in range(num_layers): - module = self.model.layers[layer_idx].self_attn - cls_name = module.__class__.__name__ - if not h2o_config.real_drop: - cls = H2OMixtralAttention - elif cls_name == "MixtralFlashAttention2": - cls = H2OMixtralFlashAttention2 - elif cls_name == "MixtralSdpaAttention": - cls = H2OMixtralSdpaAttention - else: - cls = H2OMixtralAttention - - self.model.layers[layer_idx].self_attn = cls( - config, - layer_idx, - h2o_config - ) - - # Initialize weights and apply final processing - self.post_init() - - self.ori_generate = self.generate - self.generate = partial(generate, self) - - - def get_input_embeddings(self): - return self.model.embed_tokens - - def set_input_embeddings(self, value): - self.model.embed_tokens = value - - def get_output_embeddings(self): - return self.lm_head - - def set_output_embeddings(self, new_embeddings): - self.lm_head = new_embeddings - - def set_decoder(self, decoder): - self.model = decoder - - def get_decoder(self): - return self.model - - @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) - # Ignore copy - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - output_router_logits: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, MoeCausalLMOutputWithPast]: - r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - - Example: - - ```python - >>> from transformers import AutoTokenizer, MixtralForCausalLM - - >>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1") - >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1") - - >>> prompt = "Hey, are you conscious? Can you talk to me?" - >>> inputs = tokenizer(prompt, return_tensors="pt") - - >>> # Generate - >>> generate_ids = model.generate(inputs.input_ids, max_length=30) - >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." - ```""" - - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_router_logits = ( - output_router_logits if output_router_logits is not None else self.config.output_router_logits - ) - - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) - outputs = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - output_router_logits=output_router_logits, - return_dict=return_dict, - ) - - hidden_states = outputs[0] - logits = self.lm_head(hidden_states) - logits = logits.float() - - loss = None - if labels is not None: - # Shift so that tokens < n predict n - shift_logits = logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - # Flatten the tokens - loss_fct = CrossEntropyLoss() - shift_logits = shift_logits.view(-1, self.config.vocab_size) - shift_labels = shift_labels.view(-1) - # Enable model parallelism - shift_labels = shift_labels.to(shift_logits.device) - loss = loss_fct(shift_logits, shift_labels) - - aux_loss = None - if output_router_logits: - aux_loss = load_balancing_loss_func( - outputs.router_logits if return_dict else outputs[-1], - self.num_experts, - self.num_experts_per_tok, - attention_mask, - ) - if labels is not None: - loss += self.router_aux_loss_coef * aux_loss.to(loss.device) # make sure to reside in the same device - - if not return_dict: - output = (logits,) + outputs[1:] - if output_router_logits: - output = (aux_loss,) + output - return (loss,) + output if loss is not None else output - - return MoeCausalLMOutputWithPast( - loss=loss, - aux_loss=aux_loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - router_logits=outputs.router_logits, - ) - - def prepare_inputs_for_generation( - self, - input_ids, - past_key_values=None, - attention_mask=None, - inputs_embeds=None, - output_router_logits=False, - **kwargs, - ): - # Omit tokens covered by past_key_values - if past_key_values is not None: - if isinstance(past_key_values, Cache): - cache_length = past_key_values.get_seq_length() - past_length = past_key_values.seen_tokens - max_cache_length = past_key_values.get_max_length() - else: - cache_length = past_length = past_key_values[0][0].shape[2] - max_cache_length = None - - # Keep only the unprocessed tokens: - # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where - # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as - # input) - if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: - input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] - # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard - # input_ids based on the past_length. - elif past_length < input_ids.shape[1]: - input_ids = input_ids[:, past_length:] - # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. - - # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. - if ( - max_cache_length is not None - and attention_mask is not None - and cache_length + input_ids.shape[1] > max_cache_length - ): - attention_mask = attention_mask[:, -max_cache_length:] - - position_ids = kwargs.get("position_ids", None) - if attention_mask is not None and position_ids is None: - # create position_ids on the fly for batch generation - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - if past_key_values: - position_ids = position_ids[:, -input_ids.shape[1] :] - - # if `inputs_embeds` are passed, we only want to use them in the 1st generation step - if inputs_embeds is not None and past_key_values is None: - model_inputs = {"inputs_embeds": inputs_embeds} - else: - model_inputs = {"input_ids": input_ids} - - model_inputs.update( - { - "position_ids": position_ids, - "past_key_values": past_key_values, - "use_cache": kwargs.get("use_cache"), - "attention_mask": attention_mask, - "output_router_logits": output_router_logits, - } - ) - return model_inputs - - @staticmethod - def _reorder_cache(past_key_values, beam_idx): - reordered_past = () - for layer_past in past_key_values: - reordered_past += ( - tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), - ) - return reordered_past diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py deleted file mode 100644 index 7473a3f091d..00000000000 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_opt.py +++ /dev/null @@ -1,621 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""PyTorch OPT model.""" -from typing import List, Optional, Tuple, Union -from functools import partial - -import torch -import torch.nn as nn -from torch.nn import CrossEntropyLoss, MSELoss - -from ..h2o import get_hh_mask, H2OKVCache -from transformers.utils import ( - is_flash_attn_greater_or_equal_2_10, - logging, - replace_return_docstrings - ) -from transformers.modeling_outputs import CausalLMOutputWithPast -from transformers.models.opt.modeling_opt import OPTPreTrainedModel, OPTModel, OPTConfig - -from ..h2o import H2OKVCache, H2OConfig, generate - -logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "OPTConfig" - -class H2OOPTAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper.""" - - def __init__( - self, - config: OPTConfig, - is_decoder: bool = False, - h2o_config: H2OConfig =- None, - ): - super().__init__() - self.config = config - self.embed_dim = config.hidden_size - self.num_heads = config.num_attention_heads - self.dropout = config.attention_dropout - self.enable_bias = config.enable_bias - - self.head_dim = self.embed_dim // self.num_heads - self.is_causal = True - - if (self.head_dim * self.num_heads) != self.embed_dim: - raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" - f" and `num_heads`: {self.num_heads})." - ) - self.scaling = self.head_dim**-0.5 - self.is_decoder = is_decoder - - self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=self.enable_bias) - self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=self.enable_bias) - self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=self.enable_bias) - self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=self.enable_bias) - - # for h2o - self.h2o_config = h2o_config - self.is_gen = False - self.real_drop = h2o_config.real_drop - self.mean = h2o_config.mean - self.local = h2o_config.local - - self.heavy_ratio = h2o_config.heavy_ratio - self.recent_ratio = h2o_config.recent_ratio - self.h2o_min_seqlen = h2o_config.h2o_min_seqlen - - self.h2o_kv_cache = H2OKVCache(self.heavy_ratio, self.recent_ratio, h2o_config.h2o_min_seqlen) - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def forward( - self, - hidden_states: torch.Tensor, - key_value_states: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - output_attentions: bool = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - """Input shape: Batch x Time x Channel""" - - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - is_cross_attention = key_value_states is not None - - bsz, tgt_len, _ = hidden_states.size() - - # get query proj - query_states = self.q_proj(hidden_states) * self.scaling - # get key, value proj - if is_cross_attention and past_key_value is not None: - # reuse k,v, cross_attentions - key_states = past_key_value[0] - value_states = past_key_value[1] - elif is_cross_attention: - # cross_attentions - key_states = self._shape(self.k_proj(key_value_states), -1, bsz) - value_states = self._shape(self.v_proj(key_value_states), -1, bsz) - elif past_key_value is not None: - # reuse k, v, self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - else: - # self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - - if self.is_decoder: - # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. - # Further calls to cross_attention layer can then reuse all cross-attention - # key/value_states (first "if" case) - # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) - - proj_shape = (bsz * self.num_heads, -1, self.head_dim) - query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) - key_states = key_states.view(*proj_shape) - value_states = value_states.view(*proj_shape) - - src_len = key_states.size(1) - attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) - - if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - raise ValueError( - f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" - f" {attn_weights.size()}" - ) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, tgt_len, src_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask - attn_weights = torch.max( - attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min, device=attn_weights.device) - ) - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - # get hh mask - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - if self.real_drop: - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights.detach().clone(), - past_key_value[0], - past_key_value[1], - mean=self.mean - ) - past_key_value = (new_key_states, new_value_states) - self.h2o_kv_cache.past_length += attn_weights.size(-2) - else: - mask = get_hh_mask( - self.heavy_ratio, - self.recent_ratio, - attn_weights.detach().clone(), - local=self.local) - attn_weights[~mask.squeeze(0)] = torch.finfo(attn_weights.dtype).min - - # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 - if attn_weights.dtype == torch.float16: - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(torch.float16) - else: - attn_weights = nn.functional.softmax(attn_weights, dim=-1) - - if layer_head_mask is not None: - if layer_head_mask.size() != (self.num_heads,): - raise ValueError( - f"Head mask for a single layer should be of size {(self.num_heads,)}, but is" - f" {layer_head_mask.size()}" - ) - attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - if output_attentions: - # this operation is a bit awkward, but it's required to - # make sure that attn_weights keeps its gradient. - # In order to do so, attn_weights have to be reshaped - # twice and have to be reused in the following - attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) - else: - attn_weights_reshaped = None - - - - attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) - - attn_output = torch.bmm(attn_probs, value_states) - - if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) - attn_output = attn_output.transpose(1, 2) - - # Use the `embed_dim` from the config (stored in the class) - # rather than `hidden_state` because `attn_output` can be - # partitioned aross GPUs when using tensor-parallelism. - attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) - - attn_output = self.out_proj(attn_output) - - return attn_output, attn_weights_reshaped, past_key_value - - -class H2OOptFlashAttention2(H2OOPTAttention): - """OPT flash attention module. - - This module inherits from `OPTAttention` as the weights of the module stays untouched. - The only required change would be on the forward pass where it needs to correctly call the public API of flash - attention and deal with padding tokens in case the input contains any of them. - """ - - # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() - - def forward( - self, - hidden_states: torch.Tensor, - key_value_states: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - output_attentions: bool = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - """Input shape: Batch x Time x Channel""" - - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - is_cross_attention = key_value_states is not None - - bsz, _, _ = hidden_states.size() - - # get query proj - query_states = self.q_proj(hidden_states) - # get key, value proj - if is_cross_attention and past_key_value is not None: - # reuse k,v, cross_attentions - key_states = past_key_value[0] - value_states = past_key_value[1] - elif is_cross_attention: - # cross_attentions - key_states = self._shape(self.k_proj(key_value_states), -1, bsz) - value_states = self._shape(self.v_proj(key_value_states), -1, bsz) - elif past_key_value is not None: - # reuse k, v, self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - else: - # self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - - if self.is_decoder: - # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. - # Further calls to cross_attention layer can then reuse all cross-attention - # key/value_states (first "if" case) - # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) - - query_length = query_states.shape[1] - tgt_len = key_states.shape[-2] - - # Flash attention requires the input to have the shape - # batch_size x seq_length x head_dim x hidden_dim - query_states = query_states.view(bsz, query_length, self.num_heads, self.head_dim) - key_states = key_states.transpose(1, 2).view(bsz, tgt_len, self.num_heads, self.head_dim) - value_states = value_states.transpose(1, 2).view(bsz, tgt_len, self.num_heads, self.head_dim) - - attn_dropout = self.dropout if self.training else 0.0 - - # In PEFT, usually we cast the layer norms in float32 for training stability reasons - # therefore the input hidden states gets silently casted in float32. Hence, we need - # cast them back in float16 just to be sure everything works as expected. - input_dtype = query_states.dtype - if input_dtype == torch.float32: - if torch.is_autocast_enabled(): - target_dtype = torch.get_autocast_gpu_dtype() - # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype - else: - target_dtype = self.q_proj.weight.dtype - - logger.warning_once( - f"The input hidden states seems to be silently casted in float32, this might be related to" - f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" - f" {target_dtype}." - ) - - query_states = query_states.to(target_dtype) - key_states = key_states.to(target_dtype) - value_states = value_states.to(target_dtype) - - # h2o - src_len = key_states.size(1) - attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) - - if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - raise ValueError( - f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" - f" {attn_weights.size()}" - ) - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, tgt_len, src_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask - attn_weights = torch.max( - attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min, device=attn_weights.device) - ) - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 - if attn_weights.dtype == torch.float16: - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(torch.float16) - else: - attn_weights = nn.functional.softmax(attn_weights, dim=-1) - - if layer_head_mask is not None: - if layer_head_mask.size() != (self.num_heads,): - raise ValueError( - f"Head mask for a single layer should be of size {(self.num_heads,)}, but is" - f" {layer_head_mask.size()}" - ) - attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - if output_attentions: - # this operation is a bit awkward, but it's required to - # make sure that attn_weights keeps its gradient. - # In order to do so, attn_weights have to be reshaped - # twice and have to be reused in the following - attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) - else: - attn_weights_reshaped = None - - if not self.is_gen: - self.h2o_kv_cache.clean_scores() - new_key_states, new_value_states = self.h2o_kv_cache( - attn_weights, - past_key_value[0], - past_key_value[1], - mean=self.mean - ) - past_key_value = (new_key_states, new_value_states) - self.h2o_kv_cache.past_length += attn_weights.size(-2) - - attn_output = self._flash_attention_forward( - query_states, key_states, value_states, attention_mask, query_length, dropout=attn_dropout - ) - - attn_weights_reshaped = attn_output.reshape(bsz, query_length, self.num_heads * self.head_dim) - attn_output = self.out_proj(attn_weights_reshaped) - - if not output_attentions: - attn_weights_reshaped = None - - return attn_output, attn_weights_reshaped, past_key_value - - -class H2OOPTForCausalLM(OPTPreTrainedModel): - _tied_weights_keys = ["lm_head.weight"] - - def __init__( - self, - config, - h2o_config: H2OConfig, - ): - super().__init__(config) - self.model = OPTModel(config) - - # the lm_head weight is automatically tied to the embed tokens weight - self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False) - - num_layers = len(self.model.decoder.layers) - for layer_idx in range(num_layers): - module = self.model.layers[layer_idx].self_attn - cls_name = module.__class__.__name__ - if not h2o_config.real_drop: - cls = H2OOPTAttention - elif cls_name == "OptFlashAttention2": - cls = H2OOptFlashAttention2 - else: - cls = H2OOPTAttention - self.model.decoder.layers[layer_idx].self_attn = cls( - config, - is_decoder=True, - h2o_config=h2o_config - ) - # Initialize weights and apply final processing - self.post_init() - - self.ori_generate = self.generate - self.generate = partial(generate, self) - - def get_input_embeddings(self): - return self.model.decoder.embed_tokens - - def set_input_embeddings(self, value): - self.model.decoder.embed_tokens = value - - def get_output_embeddings(self): - return self.lm_head - - def set_output_embeddings(self, new_embeddings): - self.lm_head = new_embeddings - - def set_decoder(self, decoder): - self.model.decoder = decoder - - def get_decoder(self): - return self.model.decoder - - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - head_mask: Optional[torch.Tensor] = None, - past_key_values: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: - - Example: - - ```python - >>> from transformers import AutoTokenizer, OPTForCausalLM - - >>> model = OPTForCausalLM.from_pretrained("facebook/opt-350m") - >>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m") - - >>> prompt = "Hey, are you conscious? Can you talk to me?" - >>> inputs = tokenizer(prompt, return_tensors="pt") - - >>> # Generate - >>> generate_ids = model.generate(inputs.input_ids, max_length=30) - >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "Hey, are you conscious? Can you talk to me?\nI'm not conscious. I'm just a little bit of a weirdo." - ```""" - - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) - outputs = self.model.decoder( - input_ids=input_ids, - attention_mask=attention_mask, - head_mask=head_mask, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - - logits = self.lm_head(outputs[0]).contiguous() - - loss = None - if labels is not None: - # move labels to correct device to enable model parallelism - labels = labels.to(logits.device) - # Shift so that tokens < n predict n - shift_logits = logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - # Flatten the tokens - loss_fct = CrossEntropyLoss() - loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1)) - - if not return_dict: - output = (logits,) + outputs[1:] - return (loss,) + output if loss is not None else output - - return CausalLMOutputWithPast( - loss=loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) - - def prepare_inputs_for_generation( - self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs - ): - if past_key_values is not None: - # past_length = past_key_values[0][0].shape[2] - past_length = self.model.decoder.layers[0].self_attn.h2o_kv_cache.past_length - - # Some generation methods already pass only the last input ID - if input_ids.shape[1] > past_length: - remove_prefix_length = past_length - else: - # Default to old behavior: keep only final ID - remove_prefix_length = input_ids.shape[1] - 1 - - input_ids = input_ids[:, remove_prefix_length:] - - # if `inputs_embeds` are passed, we only want to use them in the 1st generation step - if inputs_embeds is not None and past_key_values is None: - model_inputs = {"inputs_embeds": inputs_embeds} - else: - model_inputs = {"input_ids": input_ids} - - model_inputs.update( - { - "past_key_values": past_key_values, - "use_cache": kwargs.get("use_cache"), - "attention_mask": attention_mask, - } - ) - return model_inputs - - @staticmethod - def _reorder_cache(past_key_values, beam_idx): - reordered_past = () - for layer_past in past_key_values: - reordered_past += ( - tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), - ) - return reordered_past diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py index e4862c3aa77..a6000057926 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py @@ -5,6 +5,7 @@ def __init__(self, real_drop=True): class KVPruner: def __init__(self, prune_config) -> None: self._past_length = 0 + self.prune_kv_cache_size = None def self_attn_init(self, module): pass @@ -12,10 +13,10 @@ def self_attn_init(self, module): def prune(self, module, query_states, key_states, value_states, **kwargs): pass - def before_generate(self, model, **kwargs): + def before_generate(self, model, inputs, *args, **kwargs): self.past_length = 0 - def after_generate(self, model, **kwargs): + def after_generate(self, model, inputs, *args, **kwargs): pass def get_mask(self, model, **kwargs): diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py index e3de4a605da..e8346c7209d 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py @@ -141,6 +141,13 @@ def __call__(self, attn_score, key_states, value_states, **kwargs): # attn_score shape (bsz, num_heads, seq_len, head_dim) if len(attn_score.shape) == 3: attn_score = attn_score.unsqueeze(0) + if len(attn_score.shape) == 5: + attn_score = attn_score.reshape( + attn_score.shape[0], + attn_score.shape[1] * attn_score.shape[2], + attn_score.shape[3], + attn_score.shape[4] + ) self._update_hh_score(attn_score, mean=self.mean) # hh-selection @@ -196,6 +203,7 @@ class H2OKVPruner(KVPruner): def __init__(self, config: H2OConfig) -> None: self.config = config self.real_drop = self.config.real_drop + self.prune_kv_cache_size = None def self_attn_init(self, module): @@ -208,20 +216,24 @@ def self_attn_init(self, module): self.config.mean ) - def before_generate(self, model, **kwargs): + def before_generate(self, model, inputs, *args, **kwargs): self.past_length = 0 max_length = kwargs['max_new_tokens'] if kwargs.get('max_new_tokens') else kwargs['max_length'] + max_length += inputs.size(-1) for _, module in model.named_modules(): if "Attention" in module.__class__.__name__: if module.h2o_kv_cache.heavy_budget is None: module.h2o_kv_cache.heavy_budget = int(max_length * module.h2o_kv_cache.heavy_ratio) if module.h2o_kv_cache.recent_budget is None: module.h2o_kv_cache.recent_budget = int(max_length * module.h2o_kv_cache.recent_ratio) + if self.prune_kv_cache_size is None: + self.prune_kv_cache_size = module.h2o_kv_cache.recent_budget + module.h2o_kv_cache.heavy_budget - def after_generate(self, model, **kwargs): + def after_generate(self, model, inputs, *args, **kwargs): for _, module in model.named_modules(): if "Attention" in module.__class__.__name__: module.h2o_kv_cache.clean_scores() + self.prune_kv_cache_size = None def prune(self, module, query_states, key_states, value_states, causal_mask=None, **kwargs): attn_weights = torch.matmul(query_states, key_states.transpose(-2, -1)) / math.sqrt(module.head_dim) From 2d82bb54dba595261adfd7a737e24e86a8312824 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 06:28:33 +0000 Subject: [PATCH 48/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../models/modeling_gaudi_llama.py | 10 +++++----- .../modeling/kv_cache_compression/prune/h2o.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py index ac2cba82bcf..ab6da70bc8e 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py @@ -294,10 +294,10 @@ def __init__(self, config: LlamaConfig, layer_idx: Optional[int] = None): self.norm_factor = 1.0 / math.sqrt(self.head_dim) self._init_func = [] - + def register_init_func(self, func): self._init_func.append(func) - + def post_init(self): for func in self._init_func: func(self) @@ -946,13 +946,13 @@ def __init__( self.model.layers[layer_idx].self_attn.post_init() self.model.layers[layer_idx].self_attn.pruner = self.pruner - + # Initialize weights and apply final processing self.post_init() def _generate(*args, **kwargs): - self.pruner.before_generate(self, *args, **kwargs) - result = self.ori_generate(*args, **kwargs) + self.pruner.before_generate(self, *args, **kwargs) + result = self.ori_generate(*args, **kwargs) self.pruner.after_generate(self,*args, **kwargs) return result diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py index 802ebc495be..ed91b8ddb7b 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py +++ b/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py @@ -204,8 +204,8 @@ def __init__(self, config: H2OConfig) -> None: self.config = config self.real_drop = self.config.real_drop self.prune_kv_cache_size = None - - + + def self_attn_init(self, module): module.h2o_kv_cache = H2OKVCache( self.config.heavy_ratio, @@ -215,7 +215,7 @@ def self_attn_init(self, module): self.config.h2o_min_seqlen, self.config.mean ) - + def before_generate(self, model, inputs, *args, **kwargs): self.past_length = 0 max_length = kwargs['max_new_tokens'] if kwargs.get('max_new_tokens') else kwargs['max_length'] @@ -234,7 +234,7 @@ def after_generate(self, model, inputs, *args, **kwargs): if "Attention" in module.__class__.__name__: module.h2o_kv_cache.clean_scores() self.prune_kv_cache_size = None - + def prune(self, module, query_states, key_states, value_states, causal_mask=None, **kwargs): attn_weights = torch.matmul(query_states, key_states.transpose(-2, -1)) / math.sqrt(module.head_dim) if causal_mask is not None: # no matter the length, we just slice it From a9488b4ddc9f9346e677e64d7ecd58a11cb9ea3f Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 24 Jun 2024 21:48:15 -0400 Subject: [PATCH 49/62] update Signed-off-by: n1ck-guo --- .../text-generation/h2o/run_generation.py | 6 +++--- .../kv_cache_compression/__init__.py | 4 ++++ .../kv_cache_compression/models/__init__.py | 0 .../models/modeling_gaudi_llama.py | 8 ++++++++ .../models/modeling_llama.py | 17 ++++++----------- .../kv_cache_compression/prune/__init__.py | 0 .../kv_cache_compression/prune/base.py | 0 .../kv_cache_compression/prune/h2o.py | 2 +- 8 files changed, 22 insertions(+), 15 deletions(-) rename intel_extension_for_transformers/transformers/{modeling => }/kv_cache_compression/__init__.py (79%) rename intel_extension_for_transformers/transformers/{modeling => }/kv_cache_compression/models/__init__.py (100%) rename intel_extension_for_transformers/transformers/{modeling => }/kv_cache_compression/models/modeling_gaudi_llama.py (98%) rename intel_extension_for_transformers/transformers/{modeling => }/kv_cache_compression/models/modeling_llama.py (98%) rename intel_extension_for_transformers/transformers/{modeling => }/kv_cache_compression/prune/__init__.py (100%) rename intel_extension_for_transformers/transformers/{modeling => }/kv_cache_compression/prune/base.py (100%) rename intel_extension_for_transformers/transformers/{modeling => }/kv_cache_compression/prune/h2o.py (99%) diff --git a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py index 94ba87dd634..225275d6323 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/run_generation.py +++ b/examples/huggingface/pytorch/text-generation/h2o/run_generation.py @@ -113,7 +113,7 @@ # get optimized model if args.h2o: print('Enable Small Cache Size') - from intel_extension_for_transformers.transformers.modeling.kv_cache_compression import H2OConfig, H2OLlamaForCausalLM + from intel_extension_for_transformers.transformers.kv_cache_compression import H2OConfig, LlamaForCausalLM h2o_config = H2OConfig( heavy_ratio=args.heavy_ratio, recent_ratio=args.recent_ratio, @@ -121,9 +121,9 @@ real_drop=args.real_drop, mean=False, ) - user_model = H2OLlamaForCausalLM.from_pretrained( + user_model = LlamaForCausalLM.from_pretrained( args.model, - h2o_config=h2o_config, + prune_config=h2o_config, trust_remote_code=args.trust_remote_code) print("converted model: ", user_model) else: diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py b/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py similarity index 79% rename from intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py rename to intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py index 1e8078e9b40..98d0edde605 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/__init__.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py @@ -14,3 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from .prune.h2o import H2OConfig, H2OKVPruner +from .models.modeling_llama import LlamaForCausalLM +from .models.modeling_gaudi_llama import GaudiLlamaForCausalLM \ No newline at end of file diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py b/intel_extension_for_transformers/transformers/kv_cache_compression/models/__init__.py similarity index 100% rename from intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/__init__.py rename to intel_extension_for_transformers/transformers/kv_cache_compression/models/__init__.py diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_gaudi_llama.py similarity index 98% rename from intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py rename to intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_gaudi_llama.py index ac2cba82bcf..ed3248687a0 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_gaudi_llama.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_gaudi_llama.py @@ -417,6 +417,10 @@ def pre_attn_forward( # pruning kv cache if self.pruner.real_drop: + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask + if cache_position is not None: + causal_mask = attention_mask[:, :, cache_position, : key_states.shape[-2]] if self.layer_idx == 0: self.pruner.past_length += query_states.size(-2) new_key_states, new_value_states = self.pruner.prune( @@ -479,6 +483,10 @@ def pre_attn_forward( attn_weights = self.matmul_qk(query_states, key_states.transpose(-2, -1)) * self.norm_factor if not self.pruner.real_drop: + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask + if cache_position is not None: + causal_mask = attention_mask[:, :, cache_position, : key_states.shape[-2]] mask = self.pruner.get_mask(self, query_states, key_states, value_states, causal_mask=causal_mask) attn_weights[~mask] = torch.finfo(attn_weights.dtype).min diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py similarity index 98% rename from intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py rename to intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py index 3a961b9c1e0..ca97047f1d3 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py @@ -25,7 +25,7 @@ import torch.nn.functional as F from torch.nn import CrossEntropyLoss -from transformers.cache_utils import Cache, StaticCache +from transformers.cache_utils import Cache, StaticCache # pylint: disable=E0611 from transformers.modeling_attn_mask_utils import AttentionMaskConverter from transformers.utils import ( add_start_docstrings_to_model_forward, @@ -61,10 +61,10 @@ if version.parse(transformers.__version__) > version.parse("4.33.0"): from transformers.utils import is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available if is_flash_attn_2_available(): - from flash_attn import ( + from flash_attn import ( # pylint: disable=E0401 flash_attn_func, - flash_attn_varlen_func) # pylint: disable=E1101 - from flash_attn.bert_padding import ( + flash_attn_varlen_func) # pylint: disable=E0401 + from flash_attn.bert_padding import ( # pylint: disable=E1101 index_first_axis, pad_input, unpad_input) # pylint: disable=E1101 @@ -861,7 +861,7 @@ def prepare_inputs_for_generation( and attention_mask is not None and cache_length + input_ids.shape[1] > max_cache_length ): - attention_mask = attention_mask[:, -max_cache_length:] + attention_mask = attention_mask[:, -max_cache_length:] # pylint: disable=E1130 position_ids = kwargs.get("position_ids", None) if attention_mask is not None and position_ids is None: @@ -905,11 +905,6 @@ def _update_causal_mask( past_key_values: Cache, output_attentions: bool, ): - # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static - # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes. - # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using - # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114 - if self.config._attn_implementation == "flash_attention_2": if attention_mask is not None and 0.0 in attention_mask: return attention_mask @@ -928,7 +923,7 @@ def _update_causal_mask( inputs_embeds=input_tensor, past_key_values_length=past_seen_tokens, is_training=self.training, - ): + ): # pylint: disable=E1101 return None dtype, device = input_tensor.dtype, input_tensor.device diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/__init__.py similarity index 100% rename from intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/__init__.py rename to intel_extension_for_transformers/transformers/kv_cache_compression/prune/__init__.py diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/base.py similarity index 100% rename from intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/base.py rename to intel_extension_for_transformers/transformers/kv_cache_compression/prune/base.py diff --git a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py similarity index 99% rename from intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py rename to intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py index 802ebc495be..ffbb939237b 100644 --- a/intel_extension_for_transformers/transformers/modeling/kv_cache_compression/prune/h2o.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py @@ -153,7 +153,7 @@ def __call__(self, attn_score, key_states, value_states, **kwargs): # hh-selection mask = torch.zeros(self.hh_score.shape, dtype=attn_score.dtype).to(key_states.device) if not self.recent_budget == 0: - mask[:,:,-self.recent_budget:] = 1 + mask[:,:,-self.recent_budget:] = 1 # pylint: disable=E1130 select_hh_scores = self.hh_score[:,:,:seq_len - self.recent_budget] if not self.heavy_budget == 0: From a6d3fc7b0f781e9a5293134c70528c01d4429d19 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 01:49:45 +0000 Subject: [PATCH 50/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../transformers/kv_cache_compression/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py b/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py index 98d0edde605..6be078e6993 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py @@ -17,4 +17,4 @@ from .prune.h2o import H2OConfig, H2OKVPruner from .models.modeling_llama import LlamaForCausalLM -from .models.modeling_gaudi_llama import GaudiLlamaForCausalLM \ No newline at end of file +from .models.modeling_gaudi_llama import GaudiLlamaForCausalLM From 9698230e0cce78665153798b6c8686c0f92fe1fc Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 25 Jun 2024 23:02:39 -0400 Subject: [PATCH 51/62] pylint Signed-off-by: n1ck-guo --- .../models/modeling_gaudi_llama.py | 6 ++++-- .../kv_cache_compression/models/modeling_llama.py | 12 +++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_gaudi_llama.py b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_gaudi_llama.py index 7a7efc706e7..2da212daba5 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_gaudi_llama.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_gaudi_llama.py @@ -34,7 +34,7 @@ logger, ) -from ...modeling_gaudi.models.modeling_attn_mask_utils import( +from ...modeling.modeling_gaudi.models.modeling_attn_mask_utils import( _gaudi_prepare_4d_causal_attention_mask, ) @@ -76,7 +76,9 @@ def gaudi_llama_rmsnorm_forward(self, hidden_states): # mixed dtypes are not good for FusedRMSNorm, both inputs need to have same dtype if hidden_states.dtype != self.weight.dtype: orig_dtype = hidden_states.dtype - hidden_states = FusedRMSNorm.apply(hidden_states.to(self.weight.dtype), self.weight, self.variance_epsilon) + hidden_states = FusedRMSNorm.apply( + hidden_states.to(self.weight.dtype), self.weight, self.variance_epsilon + ) return hidden_states.to(orig_dtype) else: hidden_states = FusedRMSNorm.apply(hidden_states, self.weight, self.variance_epsilon) diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py index ca97047f1d3..cbb4de50173 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py @@ -63,8 +63,8 @@ if is_flash_attn_2_available(): from flash_attn import ( # pylint: disable=E0401 flash_attn_func, - flash_attn_varlen_func) # pylint: disable=E0401 - from flash_attn.bert_padding import ( # pylint: disable=E1101 + flash_attn_varlen_func) # pylint: disable=E1101 + from flash_attn.bert_padding import ( # pylint: disable=E0401 index_first_axis, pad_input, unpad_input) # pylint: disable=E1101 @@ -876,7 +876,8 @@ def prepare_inputs_for_generation( model_inputs = {"inputs_embeds": inputs_embeds} else: # The `contiguous()` here is necessary to have a static stride during decoding. torchdynamo otherwise - # recompiles graphs as the stride of the inputs is a guard. Ref: https://github.com/huggingface/transformers/pull/29114 + # recompiles graphs as the stride of the inputs is a guard. + # Ref: https://github.com/huggingface/transformers/pull/29114 # TODO: use `next_tokens` directly instead. model_inputs = {"input_ids": input_ids.contiguous()} @@ -917,7 +918,8 @@ def _update_causal_mask( using_static_cache = isinstance(past_key_values, StaticCache) # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward - if self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions: + if self.config._attn_implementation == "sdpa" \ + and not using_static_cache and not output_attentions: # pylint: disable=E1101 if AttentionMaskConverter._ignore_causal_mask_sdpa( attention_mask, inputs_embeds=input_tensor, @@ -968,6 +970,6 @@ def _update_causal_mask( # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. # Details: https://github.com/pytorch/pytorch/issues/110213 - causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) # pylint: disable=E1120 return causal_mask From 14f5a6d6039f41ba35c3a62b7b20f61de00550c1 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Wed, 26 Jun 2024 21:13:14 -0400 Subject: [PATCH 52/62] pylint Signed-off-by: n1ck-guo --- .../kv_cache_compression/models/modeling_llama.py | 5 ++--- .../transformers/kv_cache_compression/prune/h2o.py | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py index cbb4de50173..0a2ee026c9c 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py @@ -50,7 +50,6 @@ CausalLMOutputWithPast, ) -from intel_extension_for_transformers.transformers.modeling.modeling_gaudi import adapt_transformers_to_gaudi from ..prune import PruneConfig, H2OConfig logger = logging.get_logger(__name__) @@ -920,12 +919,12 @@ def _update_causal_mask( # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward if self.config._attn_implementation == "sdpa" \ and not using_static_cache and not output_attentions: # pylint: disable=E1101 - if AttentionMaskConverter._ignore_causal_mask_sdpa( + if AttentionMaskConverter._ignore_causal_mask_sdpa( # pylint: disable=E1101 attention_mask, inputs_embeds=input_tensor, past_key_values_length=past_seen_tokens, is_training=self.training, - ): # pylint: disable=E1101 + ): return None dtype, device = input_tensor.dtype, input_tensor.device diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py index 8b56bcb4236..99df3584c4c 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py @@ -72,7 +72,11 @@ def get_hh_mask(heavy_budget_ratio, recent_budget_ratio, attn_weights, local=Tru zeros = torch.zeros_like(tmp_sum, dtype=torch.bool) mask_bottom = zeros.scatter(-1, tmp_topk, True).unsqueeze(2) - mask_bottom = mask_bottom.expand(mask_bottom.shape[0], mask_bottom.shape[1], attn_weights.shape[-2], mask_bottom.shape[-1]) + mask_bottom = mask_bottom.expand( + mask_bottom.shape[0], + mask_bottom.shape[1], + attn_weights.shape[-2], + mask_bottom.shape[-1]) else: mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool) From dd6ee3cd2944f00a7eb1c92ee3fbe2b8ac4dc48f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 27 Jun 2024 01:14:16 +0000 Subject: [PATCH 53/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../transformers/kv_cache_compression/models/modeling_llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py index 0a2ee026c9c..1c8928ce4d1 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py @@ -924,7 +924,7 @@ def _update_causal_mask( inputs_embeds=input_tensor, past_key_values_length=past_seen_tokens, is_training=self.training, - ): + ): return None dtype, device = input_tensor.dtype, input_tensor.device From d241c2554e87809c725cab2ee1de5814aae33a6c Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Sun, 14 Jul 2024 22:41:48 -0400 Subject: [PATCH 54/62] add desc to h2o in readme Signed-off-by: n1ck-guo --- .../pytorch/text-generation/h2o/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/examples/huggingface/pytorch/text-generation/h2o/README.md b/examples/huggingface/pytorch/text-generation/h2o/README.md index 6f198d60b33..2a251344746 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/README.md +++ b/examples/huggingface/pytorch/text-generation/h2o/README.md @@ -1,5 +1,17 @@ # H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models -Code for the paper "**H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models**" + +**Heavy-Hitter Oracal (H2O)** is a novel approach for implementing the KV cache wihich significantly reduces memory footprint. + +This methods base on the fact that the accumulated attention scores of all tokens in attention blocks adhere to a power-law distribution. It suggests that there exists a small set of influential tokens that are critical during generation, named heavy-hitters (H2). H2 provides an opportunity to step away from the combinatorial search problem and identify an eviction policy that maintains accuracy. + +H2O can dynamically retains the balance of recent and H2 tokens. Significantly increase model throughput while ensuring accuracy. + + +For more info, please refer to the paper [H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models](https://arxiv.org/pdf/2306.14048). + + +![](./imgs/1.png) + ## Usage and Examples ### Evaluation on tasks from [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness) framework From 0c547c5f8a1af8969848cf1c1224e40a84a6673b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 02:49:21 +0000 Subject: [PATCH 55/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/huggingface/pytorch/text-generation/h2o/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/huggingface/pytorch/text-generation/h2o/README.md b/examples/huggingface/pytorch/text-generation/h2o/README.md index 2a251344746..22064c00b29 100644 --- a/examples/huggingface/pytorch/text-generation/h2o/README.md +++ b/examples/huggingface/pytorch/text-generation/h2o/README.md @@ -1,6 +1,6 @@ # H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models -**Heavy-Hitter Oracal (H2O)** is a novel approach for implementing the KV cache wihich significantly reduces memory footprint. +**Heavy-Hitter Oracal (H2O)** is a novel approach for implementing the KV cache which significantly reduces memory footprint. This methods base on the fact that the accumulated attention scores of all tokens in attention blocks adhere to a power-law distribution. It suggests that there exists a small set of influential tokens that are critical during generation, named heavy-hitters (H2). H2 provides an opportunity to step away from the combinatorial search problem and identify an eviction policy that maintains accuracy. From 372315853bb126472bcba96ab730beebfa3ef582 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Mon, 15 Jul 2024 01:36:56 -0400 Subject: [PATCH 56/62] add doc for h2o Signed-off-by: n1ck-guo --- docs/h2o.md | 49 ++++++++++++++++++ docs/imgs/h2o.png | Bin 0 -> 360013 bytes .../pytorch/text-generation/h2o/imgs/1.png | Bin 0 -> 360013 bytes 3 files changed, 49 insertions(+) create mode 100644 docs/h2o.md create mode 100644 docs/imgs/h2o.png create mode 100644 examples/huggingface/pytorch/text-generation/h2o/imgs/1.png diff --git a/docs/h2o.md b/docs/h2o.md new file mode 100644 index 00000000000..4d73c2005f2 --- /dev/null +++ b/docs/h2o.md @@ -0,0 +1,49 @@ +# H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models +1. [Introduction](#introduction) +2. [Usage](#usage) + +## Introduction +**Heavy-Hitter Oracal (H2O)** is a novel approach for implementing the KV cache wihich significantly reduces memory footprint. + +This methods base on the fact that the accumulated attention scores of all tokens in attention blocks adhere to a power-law distribution. It suggests that there exists a small set of influential tokens that are critical during generation, named heavy-hitters (H2). H2 provides an opportunity to step away from the combinatorial search problem and identify an eviction policy that maintains accuracy. + +H2O can dynamically retains the balance of recent and H2 tokens. Significantly increase model throughput while ensuring accuracy. + + +For more info, please refer to the paper [H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models](https://arxiv.org/pdf/2306.14048). + + +![](./imgs/h2o.png) + + +## Usage +Using simulation mode +```python +from intel_extension_for_transformers.transformers.kv_cache_compression import H2OConfig, LlamaForCausalLM +h2o_config = H2OConfig( + heavy_ratio=heavy_ratio, + recent_ratio=recent_ratio, + h2o_min_seqlen=h2o_min_seqlen, + real_drop=False, +) +user_model = LlamaForCausalLM.from_pretrained( + args.model, + prune_config=h2o_config, + trust_remote_code=args.trust_remote_code) +``` +To run the real_drop mode +```python +from intel_extension_for_transformers.transformers.kv_cache_compression import H2OConfig, LlamaForCausalLM +h2o_config = H2OConfig( + heavy_ratio=heavy_ratio, + recent_ratio=recent_ratio, + h2o_min_seqlen=h2o_min_seqlen, + real_drop=True, +) +user_model = LlamaForCausalLM.from_pretrained( + args.model, + prune_config=h2o_config, + trust_remote_code=args.trust_remote_code) +``` + +Please refer to [h2o example](../examples/huggingface/pytorch/text-generation/h2o/run_generation.py) for the details. \ No newline at end of file diff --git a/docs/imgs/h2o.png b/docs/imgs/h2o.png new file mode 100644 index 0000000000000000000000000000000000000000..3cd5c8ff156d77bee799db6ea56dce80e09f635d GIT binary patch literal 360013 zcmaHS18`p?Jxy7ad;SP7!VK;cu5HnB@hruNe~b)NodILH>LFJpx?z$I|+415D*ln|2#jo zObFaSKnOu3MFdsc)-JMKom3W={`tCkj7}6jL;gTPLGk$6g+VokhOGFYp7VC0oztZ! zn2m>CD5)6xiyIl5j_6?qB|cV6=uKwrp>3N$uuy%gq!jVmmM$eC>0lzu@oCdEr|WrD z6G>E2MCiZUo^7y9I_S^;YGFMjO0oZshI_VQXrytm|J7bJi6JCO|GRx9O9-O;*U^54 z^^pDlh7VU~C;9jjumWCsph4sEy~DdR!I`&>Yl z-3=4E@I6uA0u}0M;DyDG6SqBgmWryXvVwwuw>JRf>+4HITzr3jKTxSG@t67@b0J~a z-~VF+1bMpx)~4$GErY19HZ}| zIt5~zCVv23Kh&UdOVnEwAg%h@Hh1D=C-z`X7RYj$HxmvN{+tfv~Aj0 zvE%6I=*Xlp`rGdhAtNIbK}eI*3&F%oNJxC7m_j4s8%ap2s5*uN2ooej7=B7Fhk`Oo zu1d5YF|e5%II?d59k+OGI7SXs-4W64I{?tkSj>VNn?2b0;sCA=)w@yghE??)VoqW> zf-HdqV>(zq*ZuFBP-{d11src|s>olRPt%_~@QX!8R+}PkI%U>+L-Bb; zp*ZdiLvm`rQn9vYph4katenr)dDo-)8sRTBvAMJ@4EMaJQBZAKA^+z2e^dert))xU zbj53w5t$mBPHXI?#U2zi-TcsXeWkmrNl+=sb1TR7_s6Pmx!Y3~s@^Fr?FeC{KsCtc zSlVE3Z^Ir4xWk`z*MMD3c08qHoBhkWlgI0enj_Se6pfllvNj-C9@i4wWRQ%mr?kU0H35keqE$tT$>WweO9ICz^@c9Dd;)eKlB9C&58$~oAX^$QEj6=`*ZavX2rKD{MFIj^Vm$}WIse5zO zmH8;PJ$ik80ZgUFp;zS8Qgm)Xka;OU_v^%p?!?H2MsmJStf;o= zPf=59b0?S+@4(PVxZd19gss?A6;=ZW`w?=Ebg6Rm)bdwE#*X9$YO2AA=aQv5Unw3! zF-_@)@6U#?XEG(2{PbwzvzPz06onMc$=F*^0)3H?^?NZF9C)9|EiC9Ly#tuae09@q zInOyW+tV*sWl7;M1#?`G&5U}Knx~E$_(=jgvFSz)t0{Vc)J9cd_~{nmT%E3n@30VW zQFwx0vB75n37OO>$lF8+woyVb;h${*vVuddogWi4Msr#3Io_ceAuhxU3?+4Utnc&e zwQg6cULTjp2TUA})AF4?zeS7OK0t4lEqz1o#Ocdnq;FqzxwHM4TG9K-H|1pA(D8;J z|H*qM)zgV0E)FlMs~+6ao33D^Q=ZEn|M zq1xhF@%z(w7lp#QQ3De`s(0X=6@R-y$_pTUaZeH`UrC{rF-Vzy4L5LBUi9y~Mip7- z{_+Ert~%bK|4<(n%Sh6F{7A?s}Dt)3PN z)pIjK4_uWTzLBxPh?Pc;jB-c{<}^QX;) zX&nyK>~yh!ujx5od}_zJ*Mr9r^|&e{J6RCP>8L(&=cM`+Q4HszRjXGUWSVCD{0S@T zY>YJQ?VBa1tJwquB7`U07Urws42ekQuWOz(?WBvBCsZ9DQ}n4n{juWY_L>;;y25hb zU6{(%>KnrZg(t&>D%gJ&cc0`Jzd)ZS)0$jPk*Kh?BZCO$kujFTXC-*fFSUg2uQoko zy~(!+e`M2;8B|RfO;t?aANq5@ZPXASwUsqK!~r*?#kE)Mn=9XnN%k4Z!fm`}=+atq8C9g>Skh=#8d>7cDWZzJTO}R9KO|Sm zVbpt0VA-=2m=6P?S`B45WRBmw+#rT#Y(KOIhD7tckKF&DX}ibqK0;LLc0$gNgu%&& z3OwR0&nfHvdn2=Cj7lH5d5IWS#ty)8SZ8rc+v(e}7OH6Feh(N{pDKRMxu5)keB1Oz zeBPMJ(X+lcHSg>{VfTVzsxoA}#UwD%mhP({r8TFF>0E;kne)q0^j!VgtDZK!%x`*$ z>Wa&=*Y9!f@{Ou)SkBhF4PjApXAUhfkt3zQ**jKxA4Va#jPfe+o8u3EvNpW=)J2E| zfp%ytM$*7!YqZ9SypocX3W(ov5pMO=;FItEf&enT-Clc!AmzqSMF2^H8JpLL=aiD` ztu}4sR2TN+Mv;tQBK;K`uj6H6T()r6lBI{um9bW+PvjkwSI?x>-&Z4e8z?hs6`dQu zZiv=GwCUcJ2Xs`Y{OQKk8i|uG=D2wSKt^WXgJzoz5j|W&bD7@>>;F=1uyelo_He`v zi_ONdb-o@r;@XKT8HFA;;jHB;OAXBiojiIxcmTGxwvOK;M-Cn06B4i(^*}T(55F+@ z`T0LSKFqAF3d_oVf)vb-I4^694@cwq0!gCA2d_3e+#eTJSDP#`0wLk|!YHzHRcmyP z-wU-uWyneR__8Uo-2wsvK)>DT5YKg|aK)dhYN=O-Fd*O?Ck`~x?G)R!KjFwZsUSJ( zc*eit$nD1AgkJMR`e?7ApkG9UA61@R^slefcBXQdWX7)GM`k$Sa73R7wzfDWy={zw zWX-0;&L9Oi`@g(mwpt7m#UwO{b1>~EEvu*06anFtUyozy!Ob@ljjs*&*U*k+btR3R z;orLD9flVy$qE$21tg&JhK3UFO zpn&1+A#aj)}j zX1C7wZ>N$5X$#a73IMYQ2j4Ml4d~z6&+V2JsZss(C(GbfIWB6ri3rO4iTUqRxtfIj zAfy}2#=@cbuGiA`Zc?8IFuXO^P~d@-k6Om-{db|MLY15L`fBL zi_kOBQh<&F852cDAvRmj7qwg$w?B|jbr0iLR7_`FeLXfDpQ`!U#t!;MJY!(g9!rv% z@bn`r(t%qD8KYHzpY;?ipIb|$`SZ5u^^y-|A>BdD)0kcvEg{P%bX0kCWAslkPxD5p zV&c|I!s5-&kV>wm6s`f0n5;f;P4y;wj6kT!!xjzx{9(D??yAbQ)(j%BVV&70Ws4|q zokL!|WPcBkGfW=SPC*g{A z4}hK4<6tFl4}L4H1@`xcLNS0nJ6dt8m6i;5r8#@?od39Ex}xr!xb=Mzry98>xbqcK z(G7{L`P&<_jWF<7hH_fF@WPN$2YuLj!?x7b#r20Ea=b62!F=yi-{zPQ%J9VGlYJ)sNLc6`B_CB6Vm1Re7E5^9haRgw_L65{&rd-DJR!miZNz<)Tt$>q=eGt@wj&~ zTY$^wl}e}C{7qrQOt4al1@<(=ikDE4@`twz3w^KMj;uzCB{6zWpmcY_daGurji3WU zPj2;UYIgL_S|+|6@GV>D4wJwkOMCrK$F&Z^ot$bFm|D+Af{t8XGzL3vN!EOVw z-d~?Bmk>qw8b~&DK z?KAj7jYYRMl-HrLI{qZ(LYqD+PE&vC@NC(6GsCIzb{Y#yI7Bu>)yChY>fI+@<2}Zd zfpe>Y44uZ|ZR;Jbjz_WgU@j@*1mmwBRTsDQ3nYPND~*K(HSpqTf5elx9%OOfopfL57|2n( z#zwH#kXP;qd$4@n-kL+5d2c!P$I@jk9S~E?X#VT=8K4CN1Lt3L!bXfHKJ{F=mK?NC zJ(*2jO9ps@)Ni~tyv^UtY(LKi4N512k{-X>#FtN+Gr z@%Z_|wPMu9FRDISnUA<@tSMMIwj%s*Tsj^;*pjc_@tK%`Pc8JV9dW2UbEOE8MZ;wKWnKN4?=Rv5k@UkQqPTRQ)qi@x zp-P%3jE5@G&NSr>j?Nioc-BIt=)st>3+cN~w{6L;Ki+>QXSrzexYXv7m*2Z>WzGjP zw=mSz(JAP**R|qsW+#hToAGX4=+4G0O&$V2x=Eos?{ZTM05rTlnm?rbx zq}g_Rs;XrxT&E@l?Pwd9N`mc1UZnuDMIh*1`$Ep6>00q*= zUc-b&%3upYSAcT${^sy7vau5IGD=slnv6ORM^SY#{F|y|j;jd@9zPM__El)v z*xJBe(tFyBEl8M8N;iyDfQB<^>EiF->}nN=b#EQ4Z(_Xu zE_#O@j>9Xx!6lNFhq?#cg&gSp7(oQVVgXB-5@2G;i|Upn-&C@;eHWeky4r&?jHG;Y z(KfI|tuS|hq8<-G0<4Ha4N{6oNx@GNYPAqV0mp(VLa7>kK$Vll+S<)Z^%yo|#(#}W zyU%xjjywN(A0Ke(o68g~cP(!hi*faVqQvJJ@~d0wi>a9bxi|tWt~?j~?WuX^Ss0OY zR#1J3{mAzNq2Z5_QiUWjg(Js04$O=j1QNz_us{Kf2t{(kEAEPG?8crHHXeQ=H>y4z z>d6{g7?Po5-_QKOxymZqmlti5stxg~VnE5$rO7(kei%JYk|@QtGjQMa#bENs#9H*^x%H@c(PI8Dz{(}NkJTh`u7c-0HI+Y7Aj}VB z&)<9@a>+9%c7cSsOrkSnbr95t@;uEVn&?;=$}2uEa6eGr=GfF+ou~QoGlIv25H7dn zU3%5e@N(}Fh{^Lvv{5wmRDdGQ|KUzInC}SqWI!1<7sW7n30Q8YQz@^;Z88J|#7E;c z+!f!@jXj4mYn$)&%A;GKD|7VB_W}|u7aGBM8NSEje&Xj?>+~1jl7!){B4nXCe~lZk zrPO&&N=37Q->&2uX#~CA2=hn_#^%kv$FUy#!lz@%eJSXqSwoUVFHr+ta<~O=%BKuq z`m96n#1-G+E-}1A+P{D~Np^ZoFF`dHiVKWa@GwQAG*^CRC$P;>xMKn|J%DaBQb-D= z4+Bm)|43rc6Yj@`EXQl|Ziu5fSmPUvzhOXM;f@5QY;=6~)_Zw&(o;c9RXzP*bOJIg)F(*ofvSFuGi*iN)YY zNj;vUc7TF{&YV9kEG>Nl6=542`pfl}|9y6~2q1xfN3B&c-(UhzA@8Os2#N|%ekS8m z8r**%zP4Ku$7o^1$d$|5KO%3cjo(vNy6nYkmfAG#1&`Nj|L&&4;&wvnXto66_tMPD zdhgji$8?Uv(m!{D-qVb~#lv#$T>Xg5czXGv3m(~^Pn*W_r{CRUC%T=b> zRS4YUcS9Ama?p(Jg+gNR(S?)gs^U_m#$a;b;0l1)Mib(lqo&tEAcnpYOxe5t=Zq)i zgw`Y!OV3^_Poz#4_x?Gb^w8Mg1>lm{{soU0_`n(;;M6)69ba$3rit)t|0dMU>hNN{ z5%S&#&|Vw8v3D85c8!U%RBim3%!+WFnKyZ<%b2T7`Ud-fI2G*hWUV!@kp!gx-a^>b(CG>>Sm5 zp4p%^^ETi1tY;(z041I(wvaRNnl4YWl-O*_QIdy#?!uLm97W+vYl-(>C0pPD?+i3N z;j+M%=X2rF_GuL)YcHc(84p0*GJTYci|+E69UGRFdxPnopdH{h2S?-wq7|TkA@e?F zu61NRo-ft!zH8%-bId>AA|0rwO=NO~Y;yg^)Ogw}L+n0?ZhoYF-!o*#D3i3>P8R42 zocBy-;{W7yF*MBvmP0A(fVwy$pK2~Oa2o^6J!2Gh!}o*az$Bn&cB#h=Cd(HK@hx-+ zS}tkRa8wl&%EU1?!Vwf9qiuAimI{hi0L-8$Ar9|!31(D0swiS;x2<@IMn%l7Us4u< z9uDJpu5(EAo{tajAiY`Rn0vSqZXD}VB!okguQ zxnkzwA={%e$39)R1(Z&LIAldPmMosCbSFNNuv$SiW|-B)o#i0!`2a!Hvbr|DH$>!! zp~4hkv(`l?JCJ;7uUt^+eJ7h8B!bMaSQRPpIy;2+Csn++0{|!Z#gFCddNB;C_W?UC zEOo0F9|Df2u!5kL1C3Y9yLlKDZo3aGR0 zq=xMT{cjv`n6AoXX*kKXM)USVF<+iN0Q!`OhX*GudJK@wcISRQ#*d5bZ>54w|j%EY;4G=sQup%>Mdg0d9|7%{5TK*4-c>M`?uto(*IOf zw2t&*E5gk9Q>i%Mi7!tKq{gFby>2yz9;5P_hD3$;gVHoXK&iWa20n#YQDDFyabM8O zH;`z?7Ou;)93-V zMNWc!SR-8*Z~0(`4*Ha@J&^&#vktH+a+C%}Ag%_apSVbpqOG<$sl|-Tb*;)Iln!k# z|J7Q#*<}-vOyd9jmQ^(mS)r8G&0X6ZfwUS>L@r!5m#;Wln@pg8dmM3!WCx|4|EZwuhJ<4D z(s(JC@=Kh4h6*2hOukP4%lAdk9y>`Vu&kymZ;dct!Ga0PX_C!L6cND^%tRXRPe%h6 zbb;BD;ud)gEZ_0)EuU7rSkW@lnI!{({6lLU+mfQDrJz0x(ZAB}i5}W^JXST@1vd6w zJGN(^nKRiJH&5-4E*}pAqy?Rm-^^N~!>=~82HY(OdU_p!J4lT$FKG&#IAv8#^LmbcT2WgE5ix>}?rmbTf&CgWt6DLOw~qJD{&Ecq1~wqT zT95^LnkKd;R&EG5pC2zCY}-y~S+WZY3v+tTJAyts%b_ETZl3pspsk_n<^@w)(h0); z@(KcuD1O6o?sD9YrrncsStU#emW9Rb*jbu^N(G=Hs42%v+66=7C(b(j9aiuQY-_in zlRZT$=Se5serImE`R4hYHrO5dT7i+VOJYZfdN$04w#>eZWeXxf`zD7-b^wL5{M zDlcMA`IbVPuik=an`)M;$qJYMD;EG}h1V65fL3cyN6iu|Vuc=n4*9>SPqO4&Muoa|#rx2qJL<~bX%J;ph8cDo*8PMr~dx|PyW`C;1l{t|I)jyLX;dDWTl|27s^2Y7Nrx%y70 zj?SlY`w-Uc{y6y!6Y0axUaR=hWHNylgPeml%Hj&lfKw&yxAFCp#3P<5b$WUAm{qBZ zsVVN6%9KGz?7pRamc0lUtex+PLX6@4LcvM)5|gOjKx zNmM`YInr*4jxAlrM0>oKGfV4}Ncjmo=q?DFP;1Y)3)gCI33BOdW8g`mxR@%6xg`tI5-&s%*snHkS_Urw3_vZmT z^|^NE?FQ@G8+(e*{fgyE#~z8!D`V0x4&E`6gg2C{sv!*k$l|9fqvO8Umc3ahjAxvn zJvp{?%QNrKksGNAyP+1($Rn3&)q|7gSyB`3xO}@o3p?a-QQ(P$mlqM zio60#)qsOP(d2XMh*U=(s%$}vCyms0nfmj~u&1a5y^o_+Oxh}jk^&g`(j@~{UMl=( zTB!EyD)wggyj$5!Df>x2aE+;DW#xTk3iwo`{@$+XFre%M9xLvc!HclF#DerH4tIhfXiwIY&4u@Fm$cj>qy;RY^Bo$ckBYC_Jd+-o*}k@LcT$PL)gY4TIMEyA)9E}Tri|Xy*D(l-X?5_S1 zQqe|QOQ@>P=)smE{hnoJW~R5@?cJ(HpD0C!$L;iQeEhs*QavUlGZR-zN(uxT5wYZV z#h;#X@THof%F4(fSPUg??I@wVT=^Wnf#G4`^8Pnusa!gm$yvYTBY=+okCKUkl7esG z{DbrfSE}SBDe*{+YNI76tEAXK1>g9w-1&lEJ8JV#PyyQvIY*?1$Pg^94zd7-_vV^KulPMC?87FtckdyD^x1Hh$;BX%F=1_Gq6wtXc<2ghfl-ld#QR zFO*G*e1;z^E|9n>2ukgF(}DeZe3&f3(6&**vTk?27z5Qdt#9P2>PBhIZXJh`VQT*+ z501U4(!^06vqs)GI{HlL4_)^3Tq3L7vKw^R_22=|_5RU$yMY<4p68g-FP{XVp)eNc zExHrWPr41yVR)xVMs@BtRi5#T@ic-Bes&h!@^*~P~jUUqrSp1)_wH~ql* zcD|L0RpA#yFZwPYT`H6QAesQvh&xUfJ4BqNO0sQJOa^{l{P)>KFE(4Mbp-Ui*^n9c ze~&U+FR}8??P?9u) z#x_N=Y%%b?cjUOm%xXDQl%3wtpj?XTT`t)z0wM43Jmu#{H4Cc=18ewFM=xw$l>XR? zsy3zsu@pCm7X9n6Cg7%oS#i9s#>{Jt z$sqvoEBy$gU)?n=_%PfC~7k&GaI$g_>%=QkP?LpUBC7FTXN zv^=8I7|l*FhAx(f50TSbtKx{=UiAUb-rXpxQ;!XkRxlVmin#zmwDap zhD>NW`e9d~E{Cb``SgGWiiwZYXWdwH_WOg0Q`Wc=66eBN$u8q}VYz7Y6L%f#xF{r~ z^wuwKJCc@Qrj(I=hUiQm=wHSG5Bw)3PiLLjZTKY;idnR|Lv)GGwf^xmgKZiFtm>xr zZyf@6%`M@dw|SEC0BgurA9=XNeyVzKU^Vl(PjNqpo7cRQy4`XRkE(Z!L`j&1`Oc5J zr4@5gwCym+WlsZ}i&Ohg*VQ^)V2jp#c=s-q8+L$Y_0eRfrjOycUC z!77#rh#ck;I-Q=O0~i+a@}V4Wk=d)%N0YnSSlf74E)f#>kYCu0<_|u~bf}yU@HU3h z;X63fbR||-mBuV^!H1soHV!hd7Hdd|FfOxfu*dFKn~!kVFZd**7K7Fg!JGVZ3873< zj*mMZM4GImjR-GR-taQyqDw>_QWpk35$_j9Z>?GU#C3eY4F84Ds&!>$vY6Z<7T=jq z2`X_!?liBLFncJtYCvL5*F@y9BjJ`0VeiF=Aq0pA+Dc%?#VM8snTY>(KNXyN${s2| z{mxyb#__7VwJDbGe}1AE^V?LN!qQV$>-mu;7Bu$KKvy%qK3GGFFIaJZA|Q z83BQDsoMRQJT{JNh%T=5a>5@`X6X!@Kn|BDYvyQ9UXy%zvewNe-JD%`GvHGXd+y65 z5kT0Fv=!E&lN!C{O5TuSePpJOE-LQIe7&GQ$6jT8XWhbNuV!md@G%W20KLxTgxp?< zhu&o^!x|o`D9g?LGq^tu4C%flGBYDKJr~k(+qGslLjM-KUJKzarvaZrwQQOCwre`n zRvvx%{XcJ(^5TW{w4k_KqD)q+H_?pn3MSaHL1Y=VA!CX~3&ZxeYjzK|#HP^L?UyyD znX0q014!Pf%Cb$t3_XC*Ui?bN~{d3(W&CjUNNiKC!s;1 z#nXM(pr?cOug#xqUtVRXv*tx-3qJKUcq3qe(9iS4JyKz;Ph+&dydJR1OjS03H%1;U z@Bmd%b@}x#4m)6@Fs|1VEIY$jB>N?byNv}id0RxoaP#dcl&&JK3lBpyfrM4+&mS|? zs3)6%WQLX_B|F}T&U)&%p|dioX%d!uaCjjMje);ARmue@3nETelEcupE*JC^9M7yK z@~#g+V<%w+Y9>*U9SRJp37&~vB^GakkLtBd$&X(~C!Ip48zji{*(%}BKI78z91c&t z@2QNfG>ZQGXBvrl4+Jc5>a`pW0*${DvY9i|2&Tv`jmYR{C$q1Q_BiYDyKGs7hh+5? zc=)xP$u72P0}gTK%3QwvOL}P)6A88aN7SigsvtzO1@n^E=tRs>9Z3gD%8CB- zUE2yGM+}G4hLMMu8zrxEJea4FdY1(N9uI$~-;{j5Pu@8Wp*a!X^!)Adf-_dwy$Zrn z7Cs6eEGszDbndUBlf~|R#32Gijaj=>^dQ3{rRXsG?Gg(Dwe*$gU1|3~kb922NSt}6 z_afF^L&Hj&F<|?eA@3c)(h6vfEYkHJZNZtNst^4;@V$)_HPcwj zIgpD9#_`(me#u>J{o~lSCTv;zi4W!6#Ksy<&W)J$k1j!Mo_@j)mm{jlbaa|a?kHNe zQlkrgzBI+%qcv^;f`HleQZSJY^5%wyfw@~4M7AhIz4=Cy*>SwC{X1#dLPd&<^DGSe zlXyDP62gWii44Msfal+mbHz%gx#6BCft~HNnMySEQ{`8D8o{)osT?}JqS11FO)Va@ z$T5bX(z(Q0y=xsq&siqJNW@92hzC60aBd+|qQj4ruJ}OBr8?thojVH=;=Oj3qoXjV zC`N4}^D)8RFWJhC+N=aqT24SNd}!YBOx?x|Lww)o`buVdL_tu}iWWYahpxt;bMAJj zJ?ZcZVN0^(&U;j$&2(<7AR(hZVT*isG&@o7cGa<#oE(C&v2kdAXig3ReoTmRiQK1A z8(m1hpoImsO9!5xTo(84<>hBb*Q8BL#&@7=Zf*`_#-b?_n+>MwsfWhz_|}b*o_@%{ zyn3@*h1&NyUJvCzspfm>9&DgOn#ee)6YZH^*pnq(4vq0;->Nv%Eu&?R_P`Np(nvnk z{nksz$=VvQ6FFbuHDQJ*4sY9MVrkym&Bp-6=aE9K<&NhUgSyz!DWnieAXjXAY66w{ z!U)^|dE?Q{m3C$NBe{tvh@(ztnpuuXNld{Wh;G*xDM8^9P zegnrS?;n5jsdG4+Lt#U81kGhS)6orgD{iH53KO)~(+aM{3{jV+$iG(rBo`YZ?{NdU zl%+gHKEJyWsijLAQ4AD$*QWSJYftr?FLP`(|4!h6SwJH3Q@au~2c7Bea)iXHJ8p?B zZB`|Q_!?k|`l%{wQZvV6r>jdVKYD^8EmJWAZOm3!`}-WqT8XP(dt_S%9S3dtjV?!Nr2lQjONw z1w%@0Piyu`ob}4>3fiuAb4m9-3blb;==2&R&!I z@TmuohS&9602F|3rFjE6-N_#jNfI=|pFMhT)w^PGzuIQhn~5Xx8_U5#9(^ZD_I zL5Td(Q${;#Wx45cfa}Y%DM2j_AY!a#`zRaP+Ba`B-*0Fp{ubC8b^RSKcwSvJvVIGv zh`yw_EY?i*+Q)n&YciW!Q1X~>pX37DUL8q8^?b1z1vO!i*flB=IG$Cs(n%B_i&wGG zCeCm0?<3(E;j#Zq@|x87euN5VW-KI7nD@a6yX6(%vh>Gso~6d_dIkRkN50{#K-K-S zBJF}lG5YD=6${n0uM_d*VEQZN;E^E*c~JW)^QYhi9vg!Pjq~g)0n)qi=Q1M zG#@HMTJ#)W80(MX&_*I{Vlm@dvvl9Kw6sD5C$Y+pM5Ahx-aA(6EbKSnIqFx^M%WHP z{vxL!so6 zyk3?VFQ3g!hKMs9q1OD+;TN6yU>*vOBA&2>W>ai`##5YYyWiehL#4dAcxpb`@G`C5 zRee^b^_|Iv-8os>U*boIj?~8^O)pQa-oNko1_nLNrRVqR{6JJbiB)0su8>@5M00kQ z5RNIsqdvGk&zQZX5L-c82E*sE)rm|%NQujJ!A;UBFLxP5H}#L383mWbAQl~yC3YO) zlFf1Ng7)1@YQyET)lPsT+bjF@^afnqE&+<1Q7QiuV0b0bBeHB$h*4@81~?-S=DPuc z??V?%znQ{l+pDy6%_rq7jILg$d*7FrWrf8P1%=<4(0eJQ036G(Pxo_sd$cs(gD4JM~sQv zyPXp@*6pZp7!+?)iQSqN>n}XxugoG5tl~Kmdd#HUe;md9sABFnue(zDgyKcB`mArc zzk$zBsdJbz+qWF^u8THH_*&ETcnPN{tMGwmT7D}^BUcOk;LdF!1DMykPO0v=F4S^| zN{66rvlzZn;S~kwTA6a1gXIACQX%p9g}g&i05L=w#-+HHe9kv*o#Y&SW22J6oC~Y= zfyb5CoGR`|fRwoqQi;9ooESyd;Ew7FftuV^d3VaSlmUSejHaq3+k8hH&_5S@hYg4h zwQ%BwXx&_?l;!;5C&N61VdeXgx&Ufp$+YV3+?pU7Yjxg3DkTi^N&XqgWX?sqsX1`k zMdS9mD_j`Wkw%q3&fI9q+pPf+kV~F#y#c`cgmu9+B1T}v~rO3hEK;^AI2Jv9icjFlBUMzA5EIN8oci9ODHF+3r&E$C1gxb zbbY|IWX%#1+Wo$pzZgde;-$gjn5}xP9XdrhE~zN%&Yc!vVSdQk2Hu1RP?$aC&2hhR z{Y6}eF*<+K>u#uG!o!^ds(#%4`tkh~t@1tSB@VHRd~0Y$JLJ_tnjL+Zon?R3irGcR z=!}koxzd$FC{il6>;^Vs@H(^0(2}>mdvg2svG3LE`BNXa_*bgO)!w}Aal@GeTfX>! z#&3bQQ)MGOKB2lpysyugFK{!oP<)El_2avZ1QxDSo>UB7_jLa%_)I`;aEq>cl)TQC zbl&8@;h1Pbf#1WUk=B(Ob)m1)EedKB1-^Se$&*}@Q(Y940LS$5K5llSt*+-MHTzb*$3_cgKcmA)>byxk82HzkqXzTnjW+8=t-e7Dtp&(@O;19#HK;=7dc`5N z?;a0%z2&R8SI%bqF}X{AQ_t5}kpcv|zCE4{0!7WT0G2*W{)>-HdssSz4lu zyyK6I?zHLTOzysS`Tcgy_jpVj0eUb&9ElG4td*t*ROadC!(A##@OLv`)y*#3ZN9Vf zbN?e9lIyN#d@;j@tWV+GgFe5o^-FSoeu9$up8_fL+F$3)YW%~9f~m#D!8u<4x9SNF?xCpUb_H5((TTVYOc??Lsj z*tXnJ-m$6~dT+kJV&haLUq|u0xIV$nag;E*ZcS{mIWzw%?8m6aKyU@`s6j?&h-u4_ zi>Kfh^VxD_$arJjj;uRb;)Ry5r?BaEPgnl-crodXH17|SA1c+@0FKPFWZZL3JS05aV8eFx0 z8ryVN#9nTj?uB0nA2Y(P^XvBaO4();Y~`d9GW^T9A_2y9J;T-P@ooY6J4!enw_s5$ z{LJ#7w=#P5_I379TzR53aVg~X;yG>X5r;70FlC0FNCo^(Lh})*OP{NiU40oy%l{(( zwf%b4UDCP{n_F$*dFQ>K*kEy-287ZXcXl+%#^qzj#BNQvi^yBcVZZY0_LkHr>|%2z zzHYV`blCZ^@e>H|Vhs*Jz+&xHwt<6heZI`28JXUnCf|?5g9cV8<1GH6S;?sq)FIP=e^iGD{H4XJ*}xaG9fG}?MSHb5{+QBH2l83c~({i~*v$?WnAa+M)v zpGg|$lKJ7S$J=n^OroIvue}JT)E>rU;DRR&YfX%bNS)Kwi-Ny$OFZdVW(Yw#HzS9F z7=I#zBdJLfF+S5fwo4o~6?+p*SkKmE6HCyq-p_g4w(u$aT=L08C9wF;PmYk|5{?J9 z`o{vEN&U0e^fsSXjQw2%@7G9{lcTvTrRP-Uzp=-T)$F8+wVY-qeK)k(Asm&NGN->W zojEOA^?ANPSeIg*+^fEyf4m%`13TDRT-JE6+APUG`*`W+6GJVu!1hx;roGX!xCMO- z_{dJhxiA?6Jas;y0D;<5D>M$HW82>9 z7y{Itego;-tJ{FSR=g9nrUMqTY~`N1R* zG%Td;y=NdCPr0UmcA;nSRu0stZV%HrH#cp87kr9h&v7i6zph0I_}Px5v-e6K5cGQ9 zaR2yx-~+d`Ck|5h55w95?f<~~A;RV;DR}V(BKmMUPX;tAr`JXJepob4MDK$HfFsot|lGz<#DgF|7r3NU>b_W7%s^V9Ec~Q zk(8q;ETc}jlm<%LD^fMPZW(FFU*qH~2ZWCL5>9vC|8VBqzsxJjyWpijB(L*)W^c}+ z?Squ4cT#qxf)NVQ){9Pd#N~CsZzvt?Uk|BBiHY~yN^tLfr?i5h6T6%Y%8&}=04wH` zXu$`=YA^(hC)1i9&*Gk)MXdC^p@zeTn}3~WOc4B<8Wf}7jjt&fT%7#0$kaHL>r34o z@ahQ4&x8!Wb1Y*Nc2x#p$@UKUdL-7_x{`P_&g66V(+=xcttUz1&WKE)wI^ezA|wn4 z4agnX9BOG3qnLO%#B9V z!O7}>y?w+v%E3jB7My;`LZPXI4}NC zkI~qVFiTY-1S;M73iq2b?BDqBqy1L5i@NFSF97opeddIGd@x*{1$AJVQczQnlg|oL zhKg1o=Ylr@l4*+b{aPWj>m&&U<6u?C1|et6#A5b){KlY4DGvZ@@|vy+A4b)c>~ zw1Za^3MC(CX5Lfet7I^qh#ahmec$kjC4HTSll77%VrMtbzB=6)d)DLqQOH?`NL(Mh^rI^d+P!-0-NXuqEZL`2LSg#2*3PU z{c&o0)h&fvS-ETzXPZnsId~)q-2+e~Gjj4B9NDpv9X_OImV*5z!RHJ3h9LvDB$Id>bu({KC{G=Jy&ttY6dEgF)!;R13NsGxAnC*=j_lY=?h|A2)aVI` zAHjS#7b?<^^JQ`+xrYwW`<{n*`@VRstEaxx31 z^=W%Q;0V8u7fZ9rdSx-WRteEy!gwVXBq&j7MOqvthRs~Sj4{1%*NNA)BS!CTjJ@MQ z@{jK2?4o0|m>n4OYD7_i&c&BLWAEaj89O-iaU!x(kGGExE;{9Ho9|myPf05kJHf5fI^`lAt8Zb!-oChXY%v&Ieq#xvMl4`;zCqZ6k2V2>_?Vm ztX3;kRaKOgm7!EB@%Q(~$;s&#=a)1!HF4s^2^@}A93?I;?v|1;#l^*>r?)!vN~MzU z@NmMy!n$l_mvkjbx};04>oO>nXf+y?fLV56HCu4VpwYO|XVTqFU%!KolTNT{$1$e# z4JF8_Z97R;O3oy5veuhfBM0H7Q=-uM5auKj9^{KbC8G9-WWs`bm>T2TN~N4+O`O+`=E&o>h>iEFVvt59l)%4vxxW7wYL8|^-5$Y<7 zC@pV74sk)LHsKp=L=zc`tCJRLuGW>Mrii3fOW0ZD#>p544r2m%gu5HlZmm^(*AVpIws#}VwQq$KYdmL4wRz>b}qTsWEFcn@Sj zL~C%u%Q+NZ(`6o4m6IQSZd%A}rnbHwi!9)(btNFo8EbeTE(R5XY$BrHc&3l)kCUP| zaec?)P#Q3*&vBu!oXRpQhmunn>)#)DgBnp~A~sp19NdpNvnI3c(5K{|+{V_U3y2sUbcq|;UYkeqfmB@j%wtHXdsl+w*O(`94rJ6_ zFW?gv!?$02#ZS9WP+VI{+JT?RJ$sC#9XomOiN}~dG9GV}s&iq!EQ0_VgBt-s{T#>1q?QD&yPs-{yQgs9Zx6%>lDF&tkH0~A~h^;+vPS+{s8sa7A}96g*6XU(Pi zG)DAe+^_@+93}YsdR#MC}ZqThdBM zN!-e*J1627(aJ}xHMkKN6oB)!`ZDTpRJ62GG$5dM@+8Ehr93PY7p*{doLua~aX!7coAD{u3;yO-|@l0?nEpgnOwG zGOp2@Y$bQk8kTH2jN~_)sgwJ(B}iKD+u#$!pgys}VIe+f6zLS7ImY3$IgE*OrY3VA-!EBDx%U)ikLiu~rNCQ2?d(g!xWNQ0 zKFIlBTJ=UJ++CeeW2LSzjhqr2@usej;J+cVBw=?rP^emI4h5wa-^l*V@sD7@h%x;1 z?UyWCwUeB(I?nE0!{^`jC;o5u64D9!KvAK=)i(&CB%9qU7PEcdS#0jz88UDL{WH&y zRZ#-B00(zOsUg*sQe4%HT{d)lO+kUiV5IZJ6%H&G3AMVFhh7lXc!c(1`u*MLKXeQ~ zFaL(Gmu)09zlQUN*74b*c=~mJoLFyNC(bGXV^{>?9!?}?m($$bh}i<@73kgk@Dqzk z-Sq>Tx9_LO?m@qygBac~gVgM*jvwupzES1EfT{QKSXu%9`Z0;Ly=%!inarlu1DJW& z1KcxfBw>bYt|ri_*#R8}@}>YrjGsZj4Tsp9x``im-bwF;eNbD=+4plYEkVN=)yJz- zb221JqNT;s`pBY)QV{-6;C{KOjNHO9nk+J|ZO`9+EgK09^+l}^sjaQX+E9nh(!_4$~Tfh+1btebq8lK^SbC1c6^q%n{|wEha0U+^N5YwdoGPyQPf&Lh3N1pV6@=}&TrFX*H#|K|8x3>KzpfZ^7^!7k4pQEax5__kT)GA!O{4l5;M2EN|aBY&r zPF~V#mTo(Pq2Cis7!u5-<3a6CT>lXa>UWZKDHsnYZRhl&&=Hz2gV){OiKtO z#JLTpk!+Nn-OceTPo|6-ijPjb;TffM4PeORNyMx@PWF-A94fez2(JKC0w}ev^d3Kw zh(&ujo|?kJv|Rdy_Gn9nNR%Jn!x>9A7L4n0TN5M~6A93SOx?tahU5V zE;z@jLx;()wt=j~ekK37ARuZqs09H*ZN%N#cvYpFsP@3qRgEBs=uOV8v;o3phu_u3 z36)aBZnt8wTn$jFR4SD1S{WVGRi?2wDHV(7Sfc$rt{QHvG6ZtRtDiIQfm*cget5g7 zIvbXwHn`Jc%w7B`wjU!_E#bQ#H*p{}hw7>V4*$G}tn@T8UwngS7mmcw*aqqf0!oDv z1Oexu?!<<-Qm|aqvsc6j=1uL(#?%t3&ZTp@sGJGCdJ z{;C~x?XV@3o6uV#jOR)3M293rZRTrCO+SLhRtiYFfHK`!i{2U zwQ}YOk~3t6zStLUle%kz|5_6z1`m3UzKcJ__GiRTOIY&bCib7mq@uQ%S&=A;XE^mI|OM_QDu|Y0KmU>DGa!F4)K}lT;lBCCOx8D>rr&K6W zDU}FU*2Bv(joEwIakvm&U_3zqeq71ir*MhmvA=!8l(I&Q-T`=b+RqhGYMcm9n9A=W zdNFLo4}8094M|7SD6K8y^q$qcot;7E`B(V;<1+|%)3yTtf&zs?KoB%|`gbF$yWgb& z+k!$v;DD*j9{vOSbCM~|&!I5C7^1on)bCE-{J@_k@7}}&IO8y1)>m1*8A(=RcU-A2 z5rwEkAquFCA@qz7M5_b=4JMb?hm;J?czL+tC_BQ(ufIb5g@>6rW*{+PzPN?xz5Vdk z-zZvh(9%*$NqI8}0!Alil)pIGQ9z+}C3^T={utAjVQX7w!0bPnL0L^HCwDF9t(8wxpNLYSKonIN-NT8F_Gmrt)jC|EIDhnw-xY~9S-cWgk)}}vb>xI30%eN0f_Q10xGSTQYnY` z6w-ZYZ@l#C%llnzBp`YS_y4{-vC(0?@yCzYlT*#v#9bUMoJ*uvV5iuP1dR)BCN+X6 zqExC;C;{0)W%ho)`{X0mr@@%J9^tiVg9&uDuqorGTP!9KG5YuA(Kr7|c-;4Vw{jyX zCo@Ptu#xlWCpn(>EN}kq0r~|x-5xp=K}B%?kxUxai$lwfvSZb1=8lggwt62sPBqi( z{z-H*c8-EX5Jc4KHXGz{U~B8JxLxZ;mTj0@B-Az=R;wKe39F@^h6agikv&0B;Nt9r zLJ&X{K@^c?J2r=mY^$TRv<|aXhK}ay$!+&72uQXj@{exj;}5>%ge`)34?M}Nk#RUt zNac##3x+9`svEBJF8Lowl&JLhM-S(*KSVQl#2OZTyMpcePf$=a^;n4(Np~x=JLiXPcXc@$0h$;6h%ZqM5!|p65F>eLDFi!RZgLdoivHwYj;z8 zwvO|uC&)KXAx`JZ^r!xa)2Mo)1`fqTbkI~)LT>gMj-Jk?$sr@z?bxqKItqe-Ql$ix z1V+UXcB>sk0jc6NTXr4e&Y|52byb6{lH{c0sN-fbD8O*r_e*+u zI>p7s0QmU$(6eXHU+mhDWtsH!befu)0Emi;!q?Zg-QU&L*7D_-Us73F$?tymJ3Ku- zIe73OpMU;2Z@&2^d3kxf^2#f?xw*CbY(+%{R;%^Wx`>E~xaoN$4Gj(K+qaMU`ua=H zcB2lNF6l~=bV--|>c!UBNL5u078%qUJwATEmo|)`ai!O&nT+eRnZ<|q^255rjESC# zmrhATP7*s#n~8sHJl%}fIQA*g;j9?4Y?Ki|isZ z$`Bn|m5OVhTI1wRxYMn1JPM);7r$8M{Prydjh)PfAD6Loldbs#JmwXVlfrvr<3DRt#vZ0#NiY9D!FdDoWJY_zzqnW%@ds&}+kh3{u zSVbFEW(NYQ_7#PKaD}7QfeWPYVZMn*J-d!iCTMsd3Uf7Q4)3Mhqc1~ag3&3u_V=$^1O+Ca zC}uwKM+S_V#QK%r^TVpm9L_AJD(?vYe*dqy`UUag%(xrO^k@V3oB|m!a{;oYj;s^g zSiS!+=>^rujxuVRY`284v{u$$`$387>}>KItti!6G+I@=phf5EOQh?q>&4&}!q|IW zqR+^QY+1F8r9Z4@@0mgxic(qj;a@QN1@YE`1YER=t8IYLx_7U}uDlu0KL*qCL)13b zVXd#bq_2^)79d$_$v&}_eaVMOyHEp;QmUJ69oqC(g|yl44ghe`HH?WfXR<3Llig>x z^ZxISvN2&aQ>RU1#?sBmf*~;~# zWRy^ydyMb?@i#nt19)L}A3E0{(RL;av?af;zc4$hEVY*8b2{@JwRRDshZoKrpejPU zb6>{Zl1p}OBN~q$jGi{1VPV?q0->&cfoQE&oJu)HwM{{|n;SZfs>6G{@wQc=)u>U3 zGIm=tjSbC!OP6i@KNlHu%?0*s{h8ig@%XlT-y&MqU?$x2BVq)@`6EGK%pb*UZ8Y1k?sk2F!SbXLF8l=Ix)Utt#e1O*5Tm zH)^HL5>$HJy}WVOLUZ{^mb~*S?`?17-nTyHsf9yv*D64&zU9si99EiIEGV49nDX#z z44E*Ut!q}YZ0V0Ao-Sa=k`M6;isZd#C*i5OP4B{~8&l^@}2vP;7j-H{Sb~N5T*LD)gG+fB$oJB`Oh!1t?yLk82 zS6N#c#h*TVfd@x)LnR8>nnnKee1?R>ZpLDj5kw8nZceS|YM1=~zz=Tis_j4RH^d~2 zA!*Z(e7|%B+fy>Asmo{Ew|__D9>^yzO~Y4z&)M zmM&YtroE@AtSe^UkDv06lP3Xxe2DJu?GrenQhCjO6mahzO>D3)JI`d%R8vM(qlBPS z;S)QO`7sU}%FmOuY6bg`Wl`N?q4GlO@Nxn98dvzD57idiU%S-=HU3e`X@88c?I z_qnasOBj_>sl>_2>89q9$g)gMP7YhPY(bV~R4NtIr%xv?F0RWqc1fEz-X&eqB^`*J zin2nob4zd_Vsh(7&z^o)Y#0&O(B4d$ItE{LJx4eG!0zliWT~FRTX$39+=od+LbxqQ zm1LvpdNIpoadgrpFV+x|L;{LN!S`2XtGOK8|$ezTao^w3(AE{vJ~D9=9oAVWi(Z_$Yo2^cVkx88Y&XXcM0&}bv)@Xvhm z_V0P}w=eL|mHVk^k#66+5OhS1pT~q=-ZbSOW5ed%6xQ3RFUw)y{&RF6GK_A%#xAzx zU!j<5Dk&_jz#?BAQ253S<)N4V%BLT{!=qDssG?cCawB^y<_M_Kgl zE4=#tQZjA+OkVgnFFmz@UV$#R=x7KXSO%G<#zt%oJLcvFnwu^Ev0fbZYVvYRv0h0! zFRE}0?!)}wzRo{C`;-^v55YsLz*3dY`ehr)s%h?&jS~bBSASoeR7wOHE)v_zl9kHy zM_BaEE4=dlGO`?jOu6TAUi|HRdimd?z61l&WAEcHAH2uIGX~|MScI`eYY!Z!y2iShNnz-TPh;-|0^>S-4+#0w@ z*98%z`eL#R>#$uhEnQHk@rW42eJ}i(Pe1#B-%js?s}>pyPqKFXcJeA)xb5U8?JEUn zuB)Z0x(-KcSl{{eXKAFN#evnKz}-KL_};zg-O>N}I6}Of5bRYH7B?WZRx$iX577!T z2$urJF8MEsY^NyuG^rVR*ButEa1NqpbT?cLO4J$?#tyqbuLh7LM-zwFFJt4e3Vfsb z(zi!P1rhSae;r-!=DZt27;%xFsKsW|Q+BcQKqk%C7)EYZ@Wx1Z%9BO`b4d8wg-vRX4L7cZjX;_wvhlpS8rcN>o* z`NlA~ZyatHliybw!bXfE-b2@EpHHDy;}a5$vj|c{Emc+< zx64Ddo69+{aT^&eY82W4di99F)gVw`p3kY188pk+f><^xDywmzCo-lNfx+%*w64TN zhoBMcG3T$QIK0m!v$>wM zLn&0d_h3k*7s}g=?AF!QUFr(aXf#AdM&j!F%av$ZSy>bp7bD9uUS3{AMMbqsVA$<; z($dn%&CTVDFTUW^sZ(tbjmVH8LkJ5Cqq@49ty{Nt_#HZ(?otBrVnOW9NMmCoix)2@ zD=P~@5J*T!VBx}rT~u3La>efMk}m0z_Q8~;maG#=oH*A6N&{hoCNMnQc=bW1@nFb= zDGZMGqV9YOYu4_gxZoJu4;2wMd?GzOwYTF=sLtNa2e1B)4>z5o@BHU@?y>od>f0Sp zqxydX>=p!+Iz3v2fW0)A(-$gec61y}G}P5%v)(9~;h?7YI2%?cQPRPms!cxh89SdB zUVe@#@tz2F3#DgIaIVqO>H8HGYSbzLLFI&piw>niqT#|ZzWmdR{CV*a0w+DfOHVz> z_WW@ap?tu=w-8a^HkLc&VGn*uS0+-g=v*d$O*huUiv?S1<0k zV>G_XI`(hgNb>1ia!>9gwa$kTL;K>U?h4?3g{U0W=TER{>mG_5q;^4tO7B6BQS*5D z_2-$`(+`!Sk)pgb3Q8=Uvr*@g`Q(k~dFQ9|^q=(BA-+I$Aif0E~(xQEgG zqVP0oZr#?%4rH|(y~Zryt$%*aHy^#uoMEw;?3J9{wSxEl@*Zna3b4ty?`=3oj$+=_ z{7xNN7Ux|$Y?{TA=of!*#vmO-IZqtka$ZT&axvAOUh>o;uW zbWsx>`0ET}f44HKipLm>o8n8)~U* zup$GJ-9mL~DV6oD-zC{Bn9Yq;*EM68+O8*pY;UHa$%fQ+y_$yBWUwsDmxjGdR}8pT zsJ$36VFW=gS{n0HSoGN!{Cp^js`@5cnwzMtC?sX$cdXi7Mqux5ozqKQF*{c*=TJdC z9p!XohlSeWQtBKE{Q4vi@9WmCOTuoo(9qa$4Y+To?9@pzD$V%C3}x)FNHnrUV_69m z771%}1J!j6IOMiHTvbX*RXsLIMv`n;%@(Sw>##edw*4*xGIqPv?ktcU7V?fHvE$I$ zONnWP#+iuzlX+myP+axutGguS)}&7Bb)wzem=reW#10#K)_5%BNZR1R-`6^rOXel8zy%5nooKfS)5$Uc% zsduKw=n3@jP-8AV$+sVU%H|{4RM$1q($Y*_3-2-+DHN^##)mxBc35VT=-P}Y)WkcKGb-8V=%dMd9^~|~5PIGxF)m9mePZT|Z zoKSXkYX8rQrM`mw8&+_*(ApjlQyB>AF`S2=f0@VbNWe+qpt1Zcsac)#A#8Q$S^LrJ z{QmDh;fR^db1yu}tTFuw^)jKn&0&|IQlnF=5Ueef<)m@GspC!Ru(wcGUDpXWzKp$V zHgKe%x}Dt>6dJs{^=IxAukqr-A-L&8nlGFq{X*Gw{FM$J9n>l%fqC9+A%0r3hoTE7*>SLd&|wql<$1H9yAuE2gBac` z6rnDkecSg@+)%`>ttn`FPh{q(K@1E+_{sD8#m%`IPmuN=JCfLr+fGAT}HA? zE}scqD(I3fX%G5UW*ubB%FX1N!8v3Q^Y5Nauu;`+`?$o6WX|~hxXYEKY+24Xi`H|x zC720gVp<1<|EJ=pV&~#7_uoZm8`w9NW5tua+kv#Z2LzDFX>`({)^-cvDNMhEs=ADQ}i=@{M5loG19l zSKsi?3-{1FKu=9hBC9qZrl!Gq`;5BAM9kQ^OzP=DV`dV|Rn*FGvMlc}St;^{D zN&U_PUJOpxVHsS zjdxT(#!eWFx1x3Evg0~Hc6)WnSnBF%wu75bAYT67NS0bsR(;CX8_(i9WE#^(^~2ZH zUSBs0?8}X%oI0IJjai9Da4Zw=d5O;!ea)YrnM-$fCB?@R*_3pY3iEBOp;Nmvc=kL7 z1equ~wU_nlzGca_OuPq8Vt{{Viv$#C_1^e+=@5`hrqI+ z(BKm}l-Xlr&?*FE$vs>x=@ zwj?r2o7+8`Y;Wf5o-J%Wc$Q|-9UpH43PGYcGmZS}TCUq5^h?bqvW2E*Gj@l7R_}qQ zS67nczbf`x4zB!!cRyZCa%v7$jhE?IY)w_1*|~}p+m0az4q@K?3y5=3UiPzUVd+Ye4jg3HrXTqDi=`YrSA)Z0qcZCd zKmE9bHHoLFZV}Mx-0^T#BempRb-k4AO6Wj&Uj@i=3RCOaHSo(>Zng zARCvw&+p&*k|NI@^z_goAm9=`fkz&gL6n!k>7A?je$@{0Yub_+)&{cntYOP(D}5*3 z&4S7C7!)8VjdU9@n}u_R;HkBevv(=Kd*TUR`RhOV^wSS{>*Z&7;oWsO^%+mUNOx39 zH6DS%xT~T3_$IztwuU_i4wAHaC7=Im8AtM~aL6r`oJ-~B<=?Y%XDVi!0~}4%);D3d zOIR%R)YVz9b0%4+%FU%pQW4Z|0C8R}T|DOh%bu`grZD?BC(m4`z(!DD4Cuv}F{21| z6_F(gd#4H>9ONY~<a5kBJhl|>G zCCYY+PHkoFy2G6@ywsgP%KkHDSUY@P0fT!8BWFz@!k|HxrPjoGhpluS{LtP^Lz5X< zslz8CngA^{oloSmPnU3{+Q`^h^BETHeQBoOt&5GSl5EbV6(Sfti0n6>r~mX3i$41c z3r9tvH5apg$2#_b#*nOC=wJDL~w9$yT4N^m5d%enq|wDv1G{-;^X5l z{jR*coQ8%5v|248At4=pkDs3(adB}dl}cn;=ER8;G&VMN>Y4KL^7!hjuP7=i!qd}} z=bwL`$&)9eQmML(WS3lCqg^WKl3#^mB+IS;I{_v(w)kX5_6^XKwiEKSG zhu(p`aq7UJ31aIoZ$Y8Q+dm9HFYQ%}PXKV(akQaUlHE>AOBJUMpPw3XVv$t9?+ZL zOo+nR7Kli;T8<@d=0K*J`==W_72MVorm}PU5ynT2!(G*J5Tw>>QHUa@Zm~qTX%U4h z?@+>Sw_&$md+u1sNzb8H))O5+keDzRtmgA=`V#1AGNQjmUp7fXmc@4Zx>6Y(ogPws z0Xxw}hZnw2qoTZqX0yzt77zLQp#R0WRu22cM1zdo z?m!e3muIgCA}-;hxNBN(_I{Aarmw%i;4zpt-%P++*?A6TgUN&NFn<&&=~S1O(c*Ai z58WG^Q+E#jl_0A7D$ z77-UIa%2aU*(q#JJVemsJ~-)|i677t=goOkXYXV6u2T#Q8;OTb*#Yih!z#5^L0_Bj zw=_~y+kjP4;pG`Ym|s^o@Lv-_#NZRbK$9m`2bS@A>TY6s_N05T7kY(_%Dgiq?nVn@eHL4pr{XI3sLM z=h(S4gsAud%$R>C^Bi^L{d))bN7l2#-<#K74#U-;Lh_4c)Y!r7&P!zbXRmSY;`PGa zDLT57#KfQ3m0F4<*~#0#n#HT#nJ{z^@e$q_yyAH1wYO*y{>nG&50HO+8=oKF2BI3z zZv9#C*h@Th*Fc;)RWu}^HU^L|M1vz^BY%BmACW!di4F6^sD_%tO!n{IO|~t9M_+q} zN2c_{P1D{6sDu_W_O9k1$|{ET4aZ=w;q<;O?8wn^$1~6H)SO-z6at{Yv-ec)oxF{A zR~{f~@n1N*<9i~adogOp0`8w@BR6j`S^0-qvBIAhUwM=ZCwFr!c?0`0Ymw|SXSaUC zXWw{e(oLSg@czB%;@HK z%`5@|oog`TW{+dpu@okb?vJm_%~mmsIQw;H*rm9{i&;5aB z$2)wnDv5$qyZ9<~7YIt+L;Er3!Ke7m+!1(b1dvTcjhV;X;fGk1n91^Yo@ejM2qI$o zGkxBj+#}_Y_Vr$}_HSUVnufe=eK2E2C*bR*@3L?I;)Q#?8&Q}fk z76d?Yw04!rtrVdSsgaCRrzxmxMD1(DUS(aSoa!;b=Rn zTdyafP%5wQrzw5uH=qZeYYvi?cZB8NEoDIDZ|M_YYDbcfbq9lf+Q<6z9eh#b zNx!>aXF!lUx5mDwC1&&l`UGxdV|Eh%`ml(eGag}Zm?ug>MOeachQux9l5Dq7TT=^q7o7Ds*R5``Sjf!Gq_VOSQ4|Rc4ZUvC;-Zw5m64H=fy3dz zWHQmSXHT@+j*F(WT5Y?eizG?x-@l){ygWieLYOscR)?Q87z{l4;Da1EaDejia*~pg zc;t~sFqur(?R1f4ncUo5{`}`Zvv1!%TwGju>ZzxA^wCFgc6RPEj$P7~BQS}*%v_Vh?#=sY^6kmJbEXpMtwU^5 zpwa123Nl4Uwz4yIB=Zu2kZN+-{^Jt%WLILB)YMj0QgG$~C#wvM9Tttl?m(7h>~;%{ zjm^lowyOwlsXWE@Wn1YRb~ll3Is^n%dS`;7yW_4+CL|&ljaGdrA=6@UAjuMTyA`|b z3hfJO^m-jCg+zUIF4;L%^bV+I=e|PXMh_+2{W2IR>fGovdKP1PZD+~ha^(K^FmXUY zR~N~zTpTs&toZl?x_Q3Nydm8%XjGT?l*39@#vXPY%R_LFVDyx!#JFEvF$y#qJsLIG zn+nLtETi7kz_FCGm_q%~=(UJ6k+wgPlrzI|>*;~HER(IvR&hM961&Zb`l=E#Pwb|| z7S7Nn3IT9 zuST!eBFaUa-IqwpnPIs0aHpj-gDuNeb27gghy9|yc2R7LU`P))91c5@1d^?u`bK+) zz;{D-5=+-c5En3wKo<=H0&2Y{u`xmD3>Aa~`C!xwmx4)))rKU=I2=~2SC%0v6=?K& zR7f=CXOLN5gTA(egXwkj9x#L;kITTDO6SJFnRhXCT?!ksYYCq+o)O(#ZYnTTJG&Fz zr#IfK(r7Nqqu6XC6kP}QO2*t+z}ip$L3i)hxocz;CbjaaHC&T_h|P&<;1SKVDWeGX z(q9G;^?FoFk>>hJ&S#&aHp+uDJCc$6PGIo=WADBL^s4H_|G)3k+w|UNGMV&DDygJU z5=>~Jh%`|^L_ms)%I>ntcURr7y6bP()vxQayPzu~3MeFy5J({*y-j*DJu{h-=`&9+ z_uk(hlV_Mo010v*F!Oo+May%z=iKu-{oJC#Ne^AuiH5xRf*N?vPhVGqgd!T~F~Z>} z@ilnGRm8GuSMu^(2iR0w!|(s!j~J8vB%fP64vz)GJq;W@UQ1sbye1y6q`w2yLEUq+`-}fEnFId2g@dbF! zS0QT zq>wUpDgNZKEWUgRlS?xP7YEP&Ue2tuzc+Cv3+GHHF9|wQv#FT5j3<72DH@=dthh7s zNb@)b!;i^I`q(8r{JlKpt=P);y+>&6S4k_L!2Cr^SvY$FS?&)v)Kn}P%=zp?Tt9CH zX^~z|pQ`1^p~KW1KSUrBL$N0_@78be$o#p?o?1bg!}Okgaz{oPD{uV*)2&T39;qUx zE7;4g<}qj5Sj^q!C@JHq52i3{ z@#V}JpH5GmmEgSV`G?AtgZ*EV8GBL|nJMm}6B~*%gSj_7#?LQ}5a?;*RC_%KI>2mo zbNQE_Wd8g)jLY?)pre?bq~}$z;tPMv;@bunPAzs1IoWC0&yG**X{Fr!wI7o=dk6Ks zCJHCbWZ@;#$V)Q6=R$HQsFOEs0Sl&A;Qk2J+T_Y+!IjrB@0DHb-?xJr@3myQ;~R$* zY|bPybMqNLcNO2MT#5E!uby;dQMsUucfTvFU{5XO>WBV;vGZ55eb-)&H?(v7WRQ&P z0&aZhIxd+#jgriBuaL2jQL&Kwzw;AjE#JoWU57Z`8zH4=BJ(a?#KO6i_EgWl1`!UnRTj67K%naoir$ z2TyOgQ^?LtKIeNQD*n!c>^#&<;jC3$wQwq_cG-yZsh9mC;o85R^CiG|*wI`eC z>MWzV;Q$T6d@f%)1*@hK3J=a)L_?vW4N72R+!Z%*?VOGLV*N4pzVZv6i1l#mjmwx) zn1r|aFza4}X`` z#s&U?>IFTN9 zt=~oAqD#4bQ8`Z2;20*oyn>SS6som+@(VMEo+}m$6N!ROgm6e5vG`)LxN)B?pPG}+ zbaj?+s{R0{A_ZKwcrq^K-B^}fzJ%MZy@bs_Sxc{;#p1;?$w{_-vt&=@Gndci_wQ75CSqd3{7WfLF~7&VK0wvF|K`WPe4R+~GQRdV z_cO0J(ugL4~QU0rP7zI`xFi-l>^rVW;JG>XMy8Cr~p#bQI7kx!jE#nGcj zfx+v!rcRwoBoaYUls|h?L)Ufc>gssmg%>cHOg!?)BV2OHCGS0fZLu(K-aM|o_FA5K z<{6G1JH~H*^BW#}>@kv)lSeF+=(IS$pVRx zMG`RzJ44X`J$?OjpQ_@WcMj991K~5g{oDT_xp68b`I$JB2)$>TI9|1n-TMyF(hn6^ zet}y*|9MtjHvP}ut8Z~+GH2B-%zJ4s^;0fq{**G#9lNQa0Xj~dJCP7O8M&9Qw4IW;Sl!mJW5y=)#YY(L3~{eR*+UuovWid<|a z2V=?$nK2=U>aC~Q_Uu3Ke_F5NOZVPGYU?Qu?>$OKL`CmD#epLym|Nl@FFgq@*iY-p z9UN-(5>tb;)E(!e$!VD+jzBcwBIz~sY zpGd$)?V-I?x8+lsm5RffMCG#2^VkE8{NO+S$nol}{MQc;^YSk}SQH(zD~tIzKFsYa zr;+qwqK=|qwOR4iZ{c76uA8bA*Rb-+g^bPdU{b@h9NWtC&pgA%whV6i>NmM}`6QgC z^DnFkHSgfLU;mz5=PD-Tcp%ipmgj%QODC*ceb1wOVfjQn7Qmgxg!z{-Gk-U`Yc}w` zNAF|pw+j=FR`|^GCbu@6`$Vs~T!vv!p?A!7tN2aZ$ zEH4AMb=ZlYa6fxq{wd#2HuLZuS5lm6Bh-14UE7XenRYFATyqKOZWEC}Ki$pysH*EA z7>Ur`R>P5lHH@EELPnAUlj5eVVj^XEZr*O(%8$PKMb;E#GIiy>EO4GXp=c&!+$CJK zd?6d__jBnr*DxVXj**UT3Rb6saPtm+`rU6*b=9?8yKDjFxyhK-5bf35`R&vH#n$!| zR^0P-?q4+n_s|x6JK6aaj4#h-|K8&~{lo+8tKQ`sMgRaH07*naR4Ab2l52Tz_FNWR zaT#OYYNC4oA9?b@MqV73hs9)P?2Hm7mt<4FznSg7`w>r^xt9AMnSjIYAiE&`d&hYC z+g};_y{|5G6YOr~*omXmxAdW^W=Os?Os;tJh7eMJ@*1RpTCN-bQ{tBCf5Jzmu&0G;-;@Y&TR`y zumYMrl{r^j#+Z%G)a-kSzk9HrKa9)6q}ZvLQOcx}Eb0%m@Xqr;;)(Wax&JFg^taV< z{OB>x1T-RQ9b4Ym&C>LF6lA62vb{Sn(!w3Q{mhT~u}kMmH_Rv3ZNcAmkPW+AC|U3s zZd^8pOq)V1?4_smCTs3$Omdc`G@_3m!#`k-z~-VI`@U+xU%E(XD;2Lwil|N!rl5J#oTIxK95|n zRYFCk!RgZ+w%HyG#T(Gp7Whrf*(QhbdmfAlhnHUr>$x~~f=uCnbYE`WznZ}Py~n;2 z4vm@sZ;#^Ok_37qz4Wxwc(&xoI#fjmsI;2qn+bBPf~F{UzPi+C)tR$d_Wj-aHSenI z+W5j9jITciZKe)+;$L_%UMj|l9sLKB+Fv>J|k_11odsEUiX^ahTZJ&f=E$faSlXgrx6sxqQUJl zuwUnLANP5n)OfzNBWi@np9p~elFXD6P3cXSp37wVWSnTgl-(ov217k$ESg2Xm(;rX_X?VacSJBxhQSG8$6~t{)ffY96gu{8);QG3Q#a_hO4sL1Alwly zo?qz!X}%oAU*d+K+YZszF?w5YRZc$a8Q0D-z{phH?X2W(^(tyq*>NUF z_IBuwpVejGu(tlD6J+>?Nv(@)EHTi+Jj?4kT~ztvx6 ziY4u<{0;jzR1A2B@zdES>WhfHC_i6FVFPG zCp`Ifq<94`=)&MJk$DAJdt8#`8O-uIwA4|yG+%;pG)em04OUZAgBc28J$b0d=G#XJ z8Eb9NHi^w58Cz#!$9i_P=gl`+Fz&zAo^eNpdv&Lp-dB!n2d%WJ6nHzben=?0d&|#z;UMg-m2IpoE>tbQP4UBM`+6$mnUpZ_SYjodW$mprseMF%8y22}0 z=u)6jo0>FYwJto23mL(=PG5NycGEa_-Hq^Y7OqKw^*GqNM`nbPr^D9v4Alpg)KuHN zc-wV;c6Wwrg-HBxuym3ZlIr%p88L&o2hx>PGC8 z9G*Pd#^fdfkFda#hTz+865_UM&kuw`i~@r_QE7n`&HJH?%{fprL9{7yZu`#XJqa`R z)uf+Abt(Pl@S!yEZY28*=3!@Ryztbo(n#XvKbxAqkFY^$kz;Ov`6&6jn9!fFb##?Z z*?C#G^ywj?;l3LtNBDm49~HVxQ|!o+Cw=HYPYbncsuUFG^*+kHW{&WJIs$K5Vp{I- zT4?Kk{mCWO-8|@VanYv1qZ1QCgGu6fzR*S0sVU&{O+XuRY`y}KOon!3U zMoY`0U;4AFIZy)GKIe}J3W&eP5=W*PW+9Wz5HJn!{X+6CF`zdE!Ew9Zn1r3xE-QjjTnko<}&%wRaz>&=~(1C&g|FUe$+d`3x&H7LzVI+|w!;QL_-twKuYsRT>>}X=zs_JEkBr z{N(Xb$G?F@E;odYv)l(!!!GV688i;Pfzp+{GN9Vl?r3q==*ju(P#MzSSN`S{R)4qg z2emPj@`@_~C<`BpTax0G5&WYswEkc~Tkm%8g9Zfc_=1LMi44l8cW$LA-usN>`}5-l!n3U3PG$I}WQ{6SQ#AToxMIuY+ziO{5iv{tirzJqWp~dc*>yHPtcLrugaOR~-R4fc2 z2{F3Jt4Lkv=0r1#mxjlUy6|&m7)M^rBHtkPGoIHB{4KtYWzEB-8)+{8zEV&R`?NZ# zdH&Rvm2~Q#!&_R@+D~a>RQ)(_UN~EI!XSonqzWn-+^NiVrrZON@2p{jW2!1YmUw~_pIlAXDZ9Y)4LAS z%C>@OgPs{}$(0U0Fx}XI<+5=}9~;h$ealhO>iO4ycz%4Yb$^BtG2PCJ%*-wl#wLxG_{+neG?E zl7H8Jp3AyJUaX;>yuMDYPQ{#fKaI|?V%_;HJN|lzIH!=Z7ZrpYW`jCOBiRRn{Qlz7 zHm@dWI-SDsT&dDb6>Eu=7oA?CeaEUTMqOF6uO1gg$iB)gXg{s&EM>*Iv*whq)pI3o zv#Dg9kHjoVm{GT;)m=?WPEzFvS;LAQKN2Tl!e`3jj(ICnhe^F$WpGD3Baca~=R zdMLqXpO)u&-sx_Tlcwd^38Ij*?$BVUpDnCfxh?c!wr-Te5TDFK~>-!4t5SR6erXTs3}ZlOqD zQkx-$C(;MU5*-avgqJ_ye-E`x0A`C5n>QZ8g>h_qB~P6+MF@Q8)YYf`h1-Yxy7N$P zPam5tT95Ng97?I$FNlw^p)j7+byrLq_9%9j7q}0!J+qZwNM)s}1M@ZJDvx($W-Zd& z-dCSmAA9jMC*~kLd_~;G#LX^_?}_nrG`j3>%!I*(6p>kr3e;r zUHXiVH=o?Cjm%&joG#YsQvd!-Y3tRT86h4p#lLHr9X%P^+HN}5o0~+GK3Vx=e!wQX zWS(7KoSLQJzx+gKB!j%wm82r2#1zuatS7sele^9I_;11GK^>*iL+yP8 zihZ#%!LO^F$Rzf~;Y);mxhAu@3SHWwndV^}bL}0Oq8a-fDdYK&CYOgZxOxjcwp!gb z95c?&fuflSbO%QLcAm+7=h@b}5Z(p$^J&mmnoP4tU%M%x3(c+~4M_*0%%&eA5AWI{ z?1(o=*S}M|x_p1KR!E6;;pp_i9)t0uFs0#Sxa&=%h0)M_ooNJ#9FGyrZ68(I5pO5W z?^p%&RNkrg9Sakp>SAvZJaLz+0)=`aD!uQ?YVpsQ9`Ej5ZjL|3lBfq?%v9Y@&CE8} z!Sc5JtgI9~%+$KmH?l+M`59->wSW3ECPj4eK!lmI7oR<0#OsN!G%@x3_)@21=f~tL zF=u?joql?|fvt@Zt%Dmmp)9PF38`QCk$o8rzFW-y0yPm3Aj;TkGR#-MXKWYZ(1z0! zvHFA&wRb2BQ=m6xT2fQ2nxk!1ORp`3{;o0Luv={^x7~f5nwqMvtBZ+^754Jt12Dy} zL)XmAjKk>=li7IKU@(y}mD4^z`}uChVzEBz?#|7Fj{xLtZ0_vD#K(&o88<`nUQ_Yj*e$n3)WRmM`|xB zSgB@{%k5Tz>)l@dT%9!m78cebC~OKEm!e5#@OgIjtow7EJdR)bc}&R%g(g6@H*xsa z7M{I+a6g^2+1eZK_bkgz>j^Zf%S?Q%jQ;jhEZN-#>KoGK#PV1qEo*{g2VcC0u7hUR z%ABy25VhBkczdE}DIkbrHH2ZQi#k=FLzDF>Jn(8>^jH`e&5{C@&Th&gnXIPjZo6dK1kJ>VOL)DC z(`h7nVx!~!EhBWOv3yZhM`+PSNV@emNDYTMd#KFYijK2;9g1XzY0VHYWJEeSA*gSU z^Y|Lc+32-?4Ck2TH8rXDz*{mo#kw(pw}~E^%fb%tt&W#}Clwo;(7MC! z^O=18D5tg7hfW*OS?Rt0z(4WIHtoY{_{*Aa%FZFn<-x`4b`tjA#-J=Gr`-FnxZ@>9 zZbJzQ>w@_oZgg93E#Ocd4ih{qw-iz#Ny}*7Lk`K>k7A|nyFD!PdH6n<3Dq0Sk# zI_GCXoX4`uK!Lcxt+3?9|9bVD4%FN7!u^tpuvtrbj zu(NI1fn#R*`-CrD=aA)6N!csj!I!9e)3~sQtk&BZWY`jJUP(jn zbOb&T8T+lFK4qUcr?>4!$3tFQ0=_k(%sgCdBqZ~y#Ct2{dF7MUZNB%;^q#k?a~{&uzg-q^b{V$5rQds{5ISc0@L`+nkBxt&9Fgg{d+qw$6dRB=2M34L zYt6_TN45PqYuNbs_~_JBNg&qO)Wm%zJWcl&1+%sSQ(RmeWU{sG^!?Of=w$|yGAJL_ zu#@ht((1t(k|nFI{uN}Z4aC#lSGj&f4&=05f79f2wB8#a(+G8#%2 zF*GFI+uJ+2ANueqAV%cFrzgQEAMA9`f40lzrRytRHthaReK9f)t1iPteb @jwv zm4ai$EEewRGW>Adc?U5()SX%~a1T(~vc;1A3Dqd@H)ZWMm;fv~cF^fF%Q zXIRKdt7zUWOL&?>;UXM&4=eUPb>7n1hEOXHbt~uWGvV!KjGEOawa=#~_4muxRd=%N zeVnd2+<$nV)LPHIGdAbN_Xq~;&E@#R-L*EnwHE{Qx2gENmvT4BBiD=69m8d{5xBMQ zY0L2CRJ<0_G*pu!iy1@ZG$=*sq8}`H`CD@xCNC+ombttxuUA%COV{%GA3lgMVO(Us zfqJdwwntiPG50skdAaWg3<7XCCM^ABpmMNuI!~XB*S{EqU6kE7z5;@osINe)p8Sd6 zK9X#BTU`V53RDa>1H#Fbt|WW{m9%2LWM%gRTG_7Jsd2PiUN<`-LcBI4-WkyC3c&|< z6suMRc_OO(LevRU z6AFYZ0`Gxu&s9-6YMc|)fF0;k_&XEys_?^qbBnSZ>7~bn$OpSf*de@0;-S_C zR!UrSO;YoupT3*0NJuJ%*!GqUXCJ%+Ta{op3SjcjHo&irobeZbyaGVeuXb zF4CBide>FS{g<|!AAgHCQJSx>%Ul$M>dV46Si@OZQEUoZlOOh!ZJ*f*w(@RaJZK_| zRr=ChU?M%J!#y_BY zA4w{WzoS;wQsJb^;59!aEyi{aJ~O3!$0o*f3)7c!Wke^P=}+6Xb&;J#{b-H#XD5E#}-~_wOLhnV2EG!G9~?QRUI< zY-LgAXyjCoEjouF(esVW^M=nNyErgF%38zVhp@}~;sO$v@t}e$MAH4Xr%C0@1ajNDd zL3t-gsD&UTm^_q3P{XE;P~y)4X+g4$d$i5$pk4lTewL2H4=(~A*^AVZpA?CMPSNuG zPv&p=L{K8J;h9#Eehi6M_5`aMwfiy9P2a%2^_x1^4<8aYi#{MqADs#Ie?)ZYNW`<^ zq2o4!CQzEu6IMiHD+=Rz&bkr^&(!bMi}yeW1MAcl4Taq!IL%JyjwwjAb&Ix{qs2IS z2rBJ*>jL+-Te3bn+NNLQCo0BAFjHP`1nPhN=qw;n@Bcr4zWl~W)khTk-(UF8|3pb| zKSa1_HfJ5=9lJq6Rj=b|d*eMy$H5UnN^- zdx?TwWPnOtSV+ZYG5hD~@qX{*M2t_H*ze1%jjY@kadF-4!6X{Z#@M}y{Eagg68%>- zbqOYxNKwf#CcM16W|Gt5eyG8rBSy@57l_6B)a7xJU8zyFXVQy%*nXa?mN4FM!oHQV zU;s2RSh$p{ebCMjm9%UwZb7;5i6vd^d7;$3NYwDXaU_T!Nhv|`_XVpwh6z#Ac;7<> z97qgJyavk_HGvhz&bv2IpcMu41BhAOxjEa2{v;SAW2Irj#OQRWg70}Dciem~{Sw~b z=+!BmL3TRr_7|>blvvkQzz;QJk?^lk2b&>PDRqKAEoI9MzU)l<4Dt~O=!13ORHXI0 zd6Q)MlMU7`?>n!lf{_n@PGwW~0O*GX{W`c0oTi+h__J1WI1E~JlEQ8`QN9FSMBdf! z{B$>oN`X$(^7p4e3;;nJ<(qXe*N%?bBx^YX7g5%N)uX+DZ0dfM3q#C4(i{U6QkhQuLy0AN7kUK zc(p+Q>iPsD*`%YIy1MIe!;XoW8Foq1=bK*LtSNGV7@thk^>2cT!MOaQo1Kgop?mj^ zjj+5msJqcj5J3+&6UN<=z|SbzNBX!+(2RxH4{wOaQNTaU0(KpcgnyfzzK@WwXv75X z52o(((_<=!EtC}58W>d}a?mg!+XY2MIB_dt<39wlE*(w0+L{r;ja2qP4h){0%F>oR zql7?NH)&Lm`rTZ}5wBX}D4vN=IB?w&J6wq6Jouw#l-}fYB(=W9dz9-73;pg zJU!-^s)+Jt6t;Qi2wNdl+G2zJ*X-fH^|znGB1P6v$GTywpPr(iJJZuc54gq02n$Xa zXKNZiQNqm`llgrKQhED;=ZipmBC7Wmg(120b(rbsPGn}_@ff*p?Cajos&Hn-xe%C0z!nx2?d@Z`PKlFzz=|UmA7vdKUSTxBQ}@}q!;#gna<%nC4VZxPf-=`8u6Cd4 z_0BX)Wp}@v!M7ht`>rtfQ~s4Jvp^KU@4`^R9;40nHkk&(Ve(>g`OyxO*CzV%OGXfB z$xOl}vXcph5NEvYBb$SVwy8@Bg+`26z`0xRQr$VQE{%;4u#{F zx)Kv?LqHUan%s*HLz60<|7I0sm_gDCg-qr{$bf%gq0osq(S|+IdT{B1Npe8x0sL?1 zj)||rc$iu{VG-mV3+ckKOQ#of)0Lrh8^MDE!f{Kdt-t0hV5{J}j<9MNF+$O3kY&-c z#!Lh;j`)M!(5+L@#!iBEa{CMS0vXV8YlJmQ zs&wUN3x$td3LCHzov41VZ=Q`rkU?5mM&ZeBYR^~l=r3IJI+&)3Bc{)9&O%h|Q(sd3 zQ{5LHQ`;^WU>SE#*E()ITK#e`5Mj?US~lkQ9M+w1>eSU{9Dz`kDf>32Pgq|516zu1 zle~gr*TLn7a8Yn0b8>Pp=YvFu-UGcVId2UlfxGYouu=gF|IQy+c#4o83rF2+Hkk40 zd8QzAB(w6XxHzLjmYkv@EG%q5sV7KCXR7W%3m1Jc&NGgw(&Tgr1Le*lGnel#=_o1T zo0iglA_sM>=}Z@^f^&T-e|!s?gLVOvjTRlBX9@@lSPC?%!iwzCH+cYoWjBvrEK(g{ z!#t#OBX{LQOb#8qn8<-0qqXFih#!yx<U`EXXJ+`$q5ZtPcS82K9>_ zxR9jiagl+4dS%m~0jj774bY*!MGY1PacC|)_(BU%nloo^Ed_-kQ)aByv>eKyoG~dW zsa;xMc_oL}!;S1hY7jh1+e&0%;j9Il23BZjF7{ZRuLQ-(Njk?rj4NSNt9TU*luMN@kBx7nx8U0D^CFSfQ7G|^#D zfZXsgX)wOH_AFw%yWnOhngo>^_{wf;VGfif5Xh|5md?o6YLgDoo7xlr@u+QX|t^m8uNZpSJ z_>V5vyY*2g;yhY^S1&&W2WfYdr80AQrMX{bMY&={CfJJB@pepv5-7?j%weS^oZZ9~ zG~=dg5$(61x{>p(;)w}-3!_DVL5e9QE5=7U`G$oDjh422nwawPkQhgIFX*mbo)$`Q z1A(yyTP7BdjPAH1dUfc(C4%(jF3=~jZb;lfn{mu`Q}2)-@x+Tr;WKK$SgbtXj7&0lnr-$2sT%dK#)tFodzip%f_jQ6Lf`EUx)^dR%WoR&!Gp4^^ zq61qhW26VrOjAoskpE(PIi(yE7iXxVb#--B>2_%b=zU*bpN8k<6bUIQR=k$GJGc2v zDJgciyqsM7VuL+WlOP!+w8Hb=2^jb>mT-|!2qSneI{N9b!~adITaz9p7mN`Wa&TBC zi6+d|-)v1cQidP3=Uzs|#&&`V;Yclln-_Z%hf7Tuii+YgGR9+BA^}c!FX5oeE$)m- zk_rk647TgS9s8ddFmnLjn4VSw%|p(&hlshk7s`zf4i5AeYAr#W0hrzi+FlKrmNGIj zC@+=e6cs`9kt9a_09}NE!a}=;8(VORuM3Kcf&J(#uM8C?YCa$4zPP&5V@qkc82<{+ z&R(5S34GO$cEaDke}kLaGwL=oHr9T;)C73?2NnJT4N3g{CHphAu!YOC50<);I_zw; zpFt%%pzO0;NQ4rLMaX{p=9^~l7_OR{8eAS1R6q$-Rk7mY;sSCpfX?M{!;mkX z2J%>$&8KNVnls{&_X()G@tN@Z!qIl+03Gv%L{~%v^0Mm=-YYQG5}A!7boI$KXH3n^ z^uU#TPJcguHz-+vrHmmw{3= z)jyv5Ra{z{jRwV&k06E3lITP5&$P55pjjXuQ13TNgM@9cULmHa2$ayAS`F%{A$uB8EYFd&83LdC<3*LQ1*q3*IX=&+$ir9N#ln8iN)p=ZE$gvvL z=kPfEOA*uEdPhc#|N6lPz_z22OTY4MVg`N%QHtKnc;nBXS8OTaAtCRC$hm>vvcP{R z&{R@_04v9Ckwpaw9h;efLp;`@prFusxH$`ObOJSSXPcZ)WZeMrLWe-Gr{#Fkh*Qv*8nVolhhRwv4jK1XZm!6W z{T|^C!3@p}s{zc8@t^{_cY1+=>xRJQWyVOJHEpm;5wo(!jgQL zh=>3rg?a5IOr$~s0iJ~}A@pSw2a4L-E2CnQ8%GIIXm#P?;eS?<*B>4p+O09LV$%T- zu&M)&wOp+vHa;G^Q3W&7r?C{JMq4yIjYWzW{hj#dM~H z145ELMgfO{pHo$Z7TP6fWmTGK34+71+#q!eQgU)pnJJkpQru-_yTA{BEz8W*-EA4^ z?PX|gvpHRD&%A2-_3O{*1)%7G$avAexcUJdxIqXo_Z3rR{1>!P4udHohq&bGczn_W z7bwbkeT{bJ(J{gH?{@g{=w<~cZuvegEo#qm^c*`X18iq%u|KDMzt=~S`o3bqU>EAD zf&6WFgm!Q-^v?9zO%-Y7Fjms?X)`ldiSW*rdkcnWenNrqJ;^xp-Cb}f)izak_oBV` zeN2##sDy*}P0YSGITb(u??iKl47xGbq}`|T&c#S(GHM+i?VCl;pTf-q84CB_6_-nV z!EA(4_L3O}I0?N^KQ4UzbvJ1Hp#Cr34rqCpDDb}jgSrRFi9-PQ{V#6+^4tHD%>O?` z0H9v{-?D%oW}HEk@c-C?njK9O{Xe(pDMSN=|36L`Ch`CIMHxiVpkfG4usCSK{mBt2 z7i)agRga;7^a}73pU>ZqkF7e!-}wvrb*yFX9G;x8v$V9f0t`1R>(!&-PAEVLv_b{7vIBO(IOD!N%RvBy$7P?`~u47&!{$>uKP*V8epIgg+mSeiB zOSH-`q$WgT)8x*m)0a>0_!t!2zE6&LS6z&fCocV`-! z_(1c)b$zdE1&fg>@|9!($JA>gvIJezIaY#HH0bQH0^Pl?1w&Av%U=+j2q!t=470#= zUbcmmiL6UMy|54w5fO8mfUj=3TaQs_A?C)#=tdi7F8vL#jFl6Pco`#N+S=NJ^YlQ? z0Qc76aP#~&pr7tT@aW1)B*66+VZ6;JK;(Ec7m1tj(LxP1e?Bx1^IKhs+S^bF{#oh8 zXL~u`X{8FxAh$>1)ffe$O%& zCVKC5G~ZRB--CFha0@6@MDPAok!?d5&S^J}`1ka5{B+MNRUtSja7v2a3=Z2LS645r zh3`wKXm<1C)dC2=Tq&X!8Y#0r>ha&gO)?H%o#AN!kkkA>lyB7cP$9}rC8u|GB^pRJfNk^hYC;?09&((x$W^u*qK8iIg6qoK+615l z45Ni0b-Gsb5vVV`1rP%OIxDj8Bej9=-3DAVAP>X3rvpS+uvsuv<9BAD{Ay`gI(1|B zc+ZWWQ9=fDcH9|Jdf=kBB)R8}r^skE-ccM_VeV_VvbZ={RMu*?`$m-k6W1(KCY3|r z+qalZS1j)`-*!T7r^8=`h5e&^iaakfYPwX@eAiDAxP@e4GjsEe+ypHq7JA~Sw}8Ng zi@v}vP#Hg<`+Ci=&UAuI2qQ#M5l_wK+O6d!wNu&DL8*KLbzM(V5`u*G|%^d*RB9)k*Ij`)TS6D4^2`#(=WbN)RfObp{ zRoSHnLMhU(knwrGI6Bs3?sZ{&1|Cek&Wg1fDae}Z_3rVp(c|4^!0@*htAG^wa#{xm z%+?ATEhpflvhwn}N4f9`IsgTJiM>td?GASamut2$yc9%Arv>aOoy$4}p9zzx9PV0# zUoNjldHP$Q&pMqE_{65kq5wYsO{WEbXllw)E?(2aST)PzgJD81@PW^=#w71Y$eQB# z@s)V};$;8sFP=_7iGEXzTV7uFt91{Sq#v`Cd6`lJbWwYxjYr}JyqJ8t(FHfSE1A^G zw7yljkwWLB)%aMRZYME^-hqOdep14n>|aO=Gwdtsw1FxS+VIHb+%D{K;}N=(q#C)d zgm!%l=iC0C{4CsSSh)6e2RkvDAb3o+fA93L54tf-=%$|pi59*K&u~jk7%=Zpd?KYOzksZVo49fe@9jX-DR|BCCv*FK)8R6hv#3g zP7=~lenI0GC|wG$6Ayl5=7|IcF20#NP@sVHF?zEF46Kvx+>sMdmom?&TYZNOn4(fs zEljk!!Rh-vfX(2$2}98fqib3tUqcu9^dev^xt-nM(sKi4T0Xt%G^<%Zb3q&$kfmUme6F3d2#7FTpE>BhUlDDi4z_B`8rvQ9Gs;?^ zC)U3&2pTb>CToHVbK`eFqTLjkfxWk4?2?|{6l6Cc_JK#r=lcFv$GfAqg0R^g&)g&Z z@5UKs^73hsF)akas%~MlB7f!pZ{L|5AXTq-9LvfqNs{$TRzefNTL~e21ftH1eBlQC zTxYXS)DdN~yP=Us>u~SHb<`IFlh;ym=z9o}A70+`FF>h64)lF`@&Y`d{&qVB=_u7| zp*EaWy%x1Y!nxantCF6bhPDcuB;ONnujL+yR>dk6KLAVU7&kXH)xF%G0`3vSWp#sP zaMRt-dB1-LD{Hy>0}=<=RXs4ONZe{*VF3so#p-p)p96!DoWB&t!UoyZ>zsB(a!A1GgM5`f}VC{=a31<@VrG%`m6w(oNT$uCMVBf|T0su1CzM zP5$~Dw%Ad|r-B^pc&$9Ok(*Q4Vu`RHKcGtM*8%@Q1w7dG{-_Sfc9JP<;V&w{{%4X8 zx_}_!a@xHCqz@2tEN-V_T}yMoQ>~lNVEHm4V`I^kN;OP(r=s!j@BsgVm71WSr#D`1 zb`7ZEtxySrhC{a59sT5>X?(oc09ZTbRVj#kn=Vlo12JuYdJWLGr2?KL1lQ_)H<1A} z!OiXMN|!Uej`{SPsn5W?6QaLs;qIN@s$Ll-?AH^JLmxbLi0;-7ASE5C_i=M3UXDo_ z71V09Bu%~05S=y9$gmCbhU;pFR{0oyw$TF^IIhsS9S8|Y<#GLxKhX!oiT{kj6b9-w z2xJhnw`azW3Z`F6*rp<{rND41 zD+WH!ykb?Q`sX%A9>iYk#AU^r?PT+z!g;+jBO{~Jb~u7Rwz!A070}?O2JDIs#8E5T z0B4z)7wij%vJL`V&>p3u!BAEKo$NKLayhD*DR;lN8oRp(tax^DQMf{-v##yYatx^f zRM%X42{){sPE1bz`<1?hgw5MuHwegT>yOtfE3NRuvG?ksY>Yxkn`G&tx+kIEWzW^z z+4k%BzPC8QWLDbL?5L{@9QpAin>+;{g|`^RQAAs`kZ-v$>B zxZ>`awkO95H97NeZs+6G&AymZ9u&ZC{{SW^GSm3{bn+bG?+=rhN6820k{+1xcxVi2 zvSH%lb^;$9l9~Cb!o}X+p2Kk;J$Z=i>Ub$szb7J?txP6^FT6;(Owc_1W%x

F@D& ztXnQ2M#jc+6zDcUvXPXM3hpvv!FG9hUz?l59Jpw`I^KGU?7QPbxnZWsMCZs;BBG## zvWaK)sX(6&L8XQ*Zu??Yqb44?1nB;1Ya{6B=*SAKE-13bgbfG?5I&(>SX}foO_r5t zucSdlLJIy!Ab{G5!#=iDBP1lGhocy8*cA%NMTdohV`O0=aocgesG}YDyShpU1cACzjza6YZgjm^epN{P11n&C#4}UQJvE63f&p zwSYyvA^6Z&$0NLtkB?bW(Zd0bbO}P<-fg>>LEWkH@4EElF2vh*f0vj@pBx;ZS&lmM zIc4-2jLyuEH=Z8x#QO^hT&Pu8az8DAG3DorIZR~P+taxf%DNo~mv(UQpC8dzxP)Po z4QgbV3}52l$QJTidB%xOV53?_yNDux56P(h*<8w9b~J2DO&TZB#cn{tXocV}=urZm z7*~d{y^;e>wmms(h5=I&M~0Y;(--`&Gmdpzi@3_2IRdipF`Tz=;P zuOx{?U*z7=BRVI=Ngi}5`tMI8Y6^+)|M^VD&60c~`tMIxX(t&x=t%KjA4gI)@&Dd6 zD5!wCLR|3w>@2JqEcM~P_a!3Qm#~ON{jXi$zn4?ii}Cxv_HO*}|H-NQ)<69UeE;Tk z_4KP(J+|eIjqh?Ng`U%PtshP%4;){8htj!qG&nM>7%_UoR{RV8)!Po#I0+bPYHGbR ziQEq!E0G%|Wo6l8CPCc@KVH3Bb;X~ap8hG3E35|$5fC+RuGLBcrlu4z10=O;m5SOO zmu{}Z!;vaRU|#{<798x=>OCLWMQ5RQw;{A!vTCk#f1ipeDeMc)$oVJ$94pzjgpS7= zGvSe%#$K5#KFIH9S#K^K8ybG8`XDdd4t)OJOxN)iy23@Z1)eahEJ^-1iIbGmFz+j{ zvn*9y>!`-{LPW#xgIw7RXd?LuQU)53+)^V7#SR#bJt&qaZDio}2sCIw21Z&>tOv9& z-8BC*&4$(0%t2R@v4;FPi|(D_bDr3Nt@?ZdpdvH^p)?|5-Co!gxHVNxjWa7KS(EzIel1zgn~efu)phoW^KU6VEKrk-AH ziIXcYZc%b@@5KX05>*H8<$3(ceSrd1mU*ID31r4=Ww&a|p^qqzH)|-~K?D!0a0_e)?rv^2FHO1uQZOD#?`nG(X#1sBB1lQf z<|^`ou%17d5-l_8L%6XY-=xy(Xo*~%7)1(ZA;14h$ zZgl$_s_KP?3`3D{#qT+&Rp0E=yp5Irl0UJ&L#O7KoCiNpDZFvpNVvXdVw5QnN{jgVvp~nVl?UDzD`DsD z9g{)vVHmmALtOju+kx|D4sUUS0SbQT@4A@lw>jr3WVXjy`dCv)(G3mfDpa|hJ3oeP zPcJ59Hp1DLKQFV4-qdOTx*QI+e6}((BXDcG=GTdc;KLTLtAHI$f0H^CK{N&>ngP|F zA%YrklhSsJLqkLJN>(-xhN9gTR8Y>~F8i-o04IGla}$a;RHVuFNN*VG_SZhlW|AKydCVn}-ld*ex_SCqKG+UNvblPaprKgLSGbaq6s= zer#|sAOb{<`?*3Twuw%iR-?@ePXP5#CCy7NN0y8tP^T6f9T9Xfi4q20qEqx{BrpT& z!7|)KV>Pe)m5z;#>>*t^;ai)>J4a9~7tEQnCV@uZAF7z23%v#DwC5dN;#_zC1XhlJ zL6H>9J=Z(?VFaVq@Td?~^lz3SANAd`NE!+?=Gz5(nM~O~o0~bi)H@Iij4`ts9ZcBn zo}RIV1!|`R)uO4ti;K8_833S~t+US9oggVuLa3^$8dW1slhz+e=glf}(*SLxw9i<1 zc*54!bjKc8nVF$S&VR-Jot#9sYe-$>*bOpHCUZJjriI9uhXc^LwPon1o7k&a^gPu$ z6UX#(e+8bIpV0Xj$IvC^>u1bAUoc)a#YMXl%@)xRnN({}DAWbWRyYBBB4GXc_7~#+ z25)R^WRRjc3q4a)_=pX{I0@aJ2s|KtzAtG@h>u^tHMl-r%F_R>K&9*2@aVP<&1 zny(j*Q3C@51yg1Wh{VGU^4Q_9$IC4UC9Lwl(E)0ucAK#?Zk8nT1$Q{8D^$z70hBIm zD-ocCv5`5hEAxu`Az$bT=SdZKaT>cgLE?i_*Zh6-_iKqyQ{2c^3>3BWS)Lfj^On1# zVsD|%lX_Mu8#w=r(}^yrGdQ@HG#QWJSJaeFV*~d}0R#XweI1=w#Kgo|`j$>kY^}Nu zc6Ra($iQhjA1?-9QvxlJv{6QmE}^2a5p?PLIkLmZ$4A#F!^qDcsa68}3KI*fy|*`L zm!r-7nr{D+D^?zCd7ox)vCzG~PK4VNys|A2$(L^ERv(`2!6#ig=aV+;_%I@Z@EA8D z{fCZTAc^uqhBxrc`1y;~>-}n*lSO*jLd!pOAu9OyLKatevE1?DoRW>C25GDp7Z%+W z@{2C4B%YA_2x9K192LZ} z#;~E06+dlt|D+>#M@?94{%!Vgc;oT9DX@53T3F~AW`@<@b$k)@Q7bijb zvh1y*r6mf2FkYhkUr10_lj}Jduuix=?W3cpFOFiYW+L}%LV0B+)C3D7FK_q@VcX6@ z;eSA#Db*Ari)jbUkCBy?RLxtN>Z3xD;^z5(x{79%p=fDnglXS%adHM*du-rjNj-tY zI2}jFkDr?|yUF@GzDWoOed2y134zn{`QV19x?3aLbcTP}^;NcC*lVAnGW)(us|mBZ zkYZ*Al|{m4kjM(6Ez*Gc!Kuv-$FkO2~l6X*xii!Z6_4V}y5jyp|-5@#L`E=R+By-F} z2Mp@vVTxF|2m zLM6=@0E^2?u^=1`JyfJZeBN=%(m6%=jD#I4+a>MR-z$F*o;WS=(~Uu3)NswpHEs`e zZ}(Tjjiz%!!*)g2F|y#Y+DIP{K6*CTtfmWYd&k%^QkImVn<}6{XwZ?*lk5#D?m1vL zXh>fSaBx36k;Q-uLrZZ->@xc8r_TDn8%B}AkU5JEAO1B6| zNQsn$fRrGh0vnJ91C;LWZV^zC1`*lhQc}7_S|nrvi%=FwiFBX;<#V2K-g}%+?>Xno z8RNOX+$lOypm036(sbdc zuJviegysjeuMg^!H{N!>!FgdA!K$Sq>QGzClwj5K;bLK(^e*q)dF8u|s(1I)##)fE zNU`2@(9pq1(J?F6=zq{HbxHX3`a-Ml z5%~=jkpKdPz)QqRn=aHsUxd_t_ADBxcN}W&-}&>=ZDE!({;`$J(!;{aX?OnAtkX@y zf24>kF}1`wie}=b>G)zK4Jj@Jyhr}d+7~xinN?qGc85F^`$DDG%dJ)-#CjxK=t38s zV=c=Z&hE1I%KiB2VtEF4$FmN{!%yOw_{qIq!xpH96mA+(lC!y9-{qYvF(HS9u)r&E z4quWIZm}FSy>a23=kns0p64ed6v`UC{AuIz)OX{yFIEmwokdf1L#8ALg5o%# zHenrSs_6Bc+MDDk;SgFviJK~=Zg=@jC#K}T6rDlAhyq5)FpVo5v<~2u5j?-Juz+l8 zEuxy}?tO4BZTxVC-Apo`LvKYiO|!VPv~%O;{tqfBXHm3G)GHbRsEk6{*w`TC+@Jk= z1NM7@ujbXfpFAO4Z`fu1x>Rvn2D^JrLV}L&(knreTZOt8r2&Uo*xL)5`nuD)ZxNI+ zU=^09Y0maV8;~^pqdfQwS2k#ud+qRCuZ9ADW_9fumYREmi|0m#e`W^J`qEcP7)QB$ z@9ib27`5DQcVaFsDY5H$wY%2cCmW~2(GMV;g-+0zqKErFNr_m+sDZI@d#wx3{DH_% zD~hwPv$7I4$5t`b$gOvq`jv)Zql~4?bHI;pOmhDdQd@cfH9kJx3@mT^0xJr}sNXU| zC&j%3r|8ikLymHZp?V*@(av;Zvf`4{@Z56PsqEaV>!W^_G_ zBUes%oX~U;*9K;{9KE8bg=U1#z`@mcxj?`1PXkS1JOExJ$%<0GZqG zabgx%Z=7-ZHs~n+B0hrz z__d}x9$MJoNj?jNz2zab1-&KMP)(Qn`}zPUqtUcy@x;Z&=VZKACn{^mUPX`DisPY1 zp3%TaqGV>4Wl+Dp?IsydH6VxU&!C!mXb1=Q?rS(UJGwk;jpKI%Zk`^wB&&1P8h&<~ z^JXUwuDS9~l_`yLYp~;Kvg_)v^DhOD(p>YYPCZ=wSWEuMe)lwDLa8iTW_wfOj|?ca zDc?;JWhAZNo%Z>oJU_a@|EfpQX)e^W{{8P-ze920c$Mds!%}cwdwcT<9ns%zf8)2U zH1veyw_{#8%fZX*LB;bJOJ&^pmz1kDU`4!~RWkA{>Zaz1*oF4ykdNL}FiLmxYz(vH zV3ao39`NuCc@FtLAO1&X!SDHJG)Wr;@|j$D{wSTN%(%f@4DCB7N!Fhs{pzz*>_R3? zW}N8SHTd_}X=!(U7=&9y_Pi3}c3*?RLD01RwSf$#3L}^1tFs>B7PwVwdfcSH1Et@|2S-`v&N0l&O0hLu^vV6bD(=oJ!KnDW{(=wx zj__$^-ORXsTvFNj<;(feu`#N&Ci}^nAYkgRyi9tFOX{Dmt*wQ-v->&S*Wh(d<7{d> zT6Yl4yKhDnU+wm)mdv8eIK;4lok_@c;KsWh;SrguQbef8EZy&ulkNMPI4Wt63l}cr zN?dST5GWBVHER+yy`#JVqa3IU7t>sTspCtD-m{A7pPH)kK$n*zPbU~v?SSos0q9~B z&WqI+^CWlmjO~j-I>8xKWaG&%2|-G#*fG;`E|0NCU(?#uI%C$p-B$mapmK#lXzAhp z%>1d^0^LD|o;JpmjVB=I?Cn3jZjFSOqHILEuVrpc9VB=+`X&#sWUVZYrWZXntK2r1 zB@s9{c(5?3%FUXu;;luj6_h(;5B$eb@ zB_kkBC%_9so-m)ajTGry&=c;gd=Ih8^itt}iSw%Q-4}s%WbyU`{rDmOpSHS=xf_*k z$@cyv*wVH;+m>!Wn20rN^|>Dz?F!d#uqTrNqaFbQ9zkJp?{ZJ8a65ok__T@JgnzeF z-JNL;!p|+kNC8xVPYGs6WF7|#NhC{8U!P|E>B%8rkngs%WsL`X=Lu?6Gwa*xB*0%Qc&_#M!-+VA%#%fURPYtx*ru+HF#qC^4#{i0@wRf9C&W#%GQP8XZa@jB@ zOQEefyq3vk!QVQku3dB0g0m*9#hmrsQ%i4e#`*dA+_DSxo*S2q$CUX}02S2-+KOLw z=Pw%J`!A$)4Pbbg->OaS{xYon4U2rGA0VUzh&|LXm5q(@uw}M5F&oHo ze2!-eO?<&rz4Jw^hRG$6ERl6GZe>p=x9dZt=Q-2IctPqGTp!t(mH-BeRG@nrLvyjE z(!lCn6A`&g3-3R}SeaSG`XN0%ee=hGn^(Q*Hd({Rr0itJm&Yg>mkB=T@g_A1S4g(6Br1#eQ#pTQJNg&pzZ3x$zB$XM`(BJ{yMLgzZNmR}51KDokMhSvR=3X9@7 zBp3D@y3C*aG^L{dL@N5^W-yttXs-hQ53&}qM?1XKq6hU{8sqxr)@-1cwL|Evqtmcr zv_zM0&~Sb*M={q$ev9w7N?wq$U<8t~s?akwGHR=|8H2~GqN)mjOW@#B2|P&)`2Mi1 zLqzp2zan`GKRg;9vMyiP@!{@6*tnf5pWDsD0r+082vBrxS7PI>ZCP?W*;p_V+%*7C z5(0Lt>z{m#S{^HF0|IGh{9*0}&;W3TA@9yRkwvTveM232bcaoGf3Ekgh;LndbFan2 zMaeUFO2>Ie9e~3*$Iui#J#C8McGtWwqMF9_o|NyN&cO?Ve04zhm4+R?y&998V3jFf zB;0_n-JgB~f6NV8Z+WT-(X`T_|7|QYAlPj#Mkw0unRjLcG^Oli5eNJs9)Cz_@SI}L;@HC_dVXm3be4$HQ8>J|`dF;IJwJ)`jKaitHuIEG# zFO5Q~GnYnB#h~F42|G!0@6_wNvR6kr(G?-FrCpjY8hm!6?Vo{IF%RSbyv1OTL+*o6 zl9y78eg!j3A5hFVKb@;^tVUtk3VtGCxiJ0NBj|!e*P6A)FSxWk5~j zirwG(5(72>nf)J0xh@09!ZfL)#ii_ov#gKDD2@FCtl4~mT>Qp{6WRdSi`S{CLBQ#4 zhJ3!u8qU5adoQTrT$v0x%bCKHb3b-vxV`xse#v^K5bW{ArSyuf{rvH3U1PjCTOuL1 zDc)5o4*9GRHqE9WHI?D4LLl;4NV(pwP`en;4rd7-V07S47|TYU6|>b)^Qw&_%l!i0p?`_b*j_;98Q9kbtaE zOU;X2PB03hD=N-DrvDtzEdOLBaGb7#sD#fj<`t?MV5X}5+@<2V=abBXJUKUf2&GWs zIvlSD*uo!vo+Qc7zV^hdtHzIpFfcKZQfB*`j^)GVUBBM`{#@-sgpC8U0obd{CXW`p zYFw8t!nx<|?Tzfhk>ZbmBXtDoxu!G*lw-sN4>Vz1O};j-zkrh{?)ICEj5vqV$)*6g zopr2Gwb_`pxw|_p%r)rcxI6l+IVl4@|bx_nVV}uIL=b-$`$NxwcT5u>s$xQ z8{yx*R;ViyZ&?%jUu#G_QzgC+vx019xP>idrHcOo=q+}EMlUN@Cjm&7_Vou#zY)%l z&mWcA@XWyRMsrto(}!Ce?Z3qmVKlv@81aGnzaGHbcG2&HBE!OHtBwN0mE=_U`jP?K z)?4rS(L&MWl$Szl1p?G@Ws44~=(lAl{aKpB8vLfBbZY&wLug&~sDP^eCIP=S*7OUn zvznxC=of}M3sPrOCtE~zPMjZaT+62T*S(*nD8fR4qt@wgMp8smr}$m8v!a#l(mrSO zE8o>1rFrEpZ9Y@_@ZMUpKeb6RW|&=4YV+4dmH+mR>xH|W^3ptx9Kc)L!!W+HHq|4p zK3?Vgm4;2`?pR0{)~H{DiyA>?9>OIh|484QIqI&TWZ7Gd`Cs(W|4~;#9sb|dl>e%$ zIhFANWWlKTd3j#Y2Z@9CtgySYlJN%oT}~9Rj(irSiDiuVO&5K36AH*I&XQ@QT#T^_ zojxA7;PK`UxfHm5xUpWk-0H*(O8vzFpnGurPRv6#!|+cJFGq8WEl7bnMvkdcmNUZD zzz$kv7n32~3+^n~QyCV8FrT)-~-ol2>_H+yfVK??~Sz`%e?+KcE29Ug- z2+23G&N)=qs1YvrD|Io^w~h74tmA0{T5HHA-^KpXO$Sw2 zZ<(Xs7@7W~qpWW(7CioI)fq9u@{4u%-c8nw+}F$(bDgPnDP3$Ii7&{WPrusnO_18v za`Q2+_Cgy)CN8QpLu%P#DIN15fZNQcVxB2H@pElVe7>$ANt+|)s3ZNbZuK7}T#<{6 z@AolF9CkDa)8C`M;Qg)fLfoNSH5v0zeR0TP0A>)ccu+a}(Ht*J>$g+1UNR$we1USQ zeq_Hosgy4G62+-+#{4D2HqL9EdHbS;E~--@nisu!z$wAqvp{Oaq9quHA zt%a@H5*>W08G5;-TdJQK9R1)nI6RWA^|ZfWZ-)|~Doobpti zBSKZ#buq;O7;FFf^@TybFbNvu1Ui}Z`b`9)-NY@M~?FLJ)uGYAsh^;GM$Q+5nFK!jJwQ2$R&C8AgTU{zhD^o*+JAX zC|1d%Ofaf;rAxVRT4vo(`X;|*p(4D&m1cV3-$nG+^6G&!i?~y^|9PsCJI>qX?)SGa z+a3+3U5D#U37fcqm`rI*)%?LBx7>ad$l_zmPDz_E;=o+>K3(Lnt?Dvd{C2-#>*|oojazIv*y&9McdHGJZEoVwDl!_;4+;YI z@mSE?n3$L>ZA?nUfbXA|@dD=-3_|x^ON>hN@KEq)2)?y08MhXlGrac2=tZSvp>Fip zx_+r~OKWTJBE6ZB5vFSTc;(}O((Y>nnQv?8QVr+DAuAHrf?D}^+c^k_MbXHByf^Ln zDt7o}okGivrhnnh77_lS|ChXi%y#rQr=ipLnRT#SYHD78#zn{HKiXVt@$7q|BV&ro zwY9So5L5x>H^XCSh)ymO_x`|fzhMeVIRit&Mos)~eC zo7?mVL{#qsw*%L)u5-T-lC~jUf*s`QQmqvvyb?E91V&+Ari!BrXE*<&*^p@6cs>{* zjdIf}G0vsyoNrw4eW-$T?r`Gxtf_-9{!zG*n9dZ=j{J^&T1zfCNiv`nsEP=rBzk^~ zX)(f4ZvTe!T7a5Er>tZE7mbj&dWPnU2s!(V<$SQxYBc1C1$Pn#KT$itw;!3ptN)U_ z!aPVh@Rf|v_`8kSlu$jDtcT(9-j(m1a|8;{txK#iRh}h#hsAYC=`Ohi&Wl4`SI)^! zo*hM_bi>cJY2dv`h$Uhz-+n_gEUH{LGemVYQE8r+piM)8!TBzSQkj)?nKJ2FJ^_!l zig3#5fQ%q*^{x1*MZsz7i@*_YO)AH{x-FGK8?b7Od&|#K=3?=#669bt&P;7}-=spL z*MCh4W!?P+{u;x#1v5n+t!LVBYb-X$%rm^$sF>a{#3v=VThKwFFcw(xf$jvKr^DFHeydc+M?DT zlR5L!G~f!$ddo!2A+?G2@FvAz8>kZist_Zh|08mD)_pmpB;}=1hGMf*3f`Xf*|vzk z!+pmGJ0>>&J$BdzqkV5CjG6k79`3AC#Z-fVuNv#m1lCm$(%o4Oo6H~oWWDcT&u_jz zS2k1q%<)5>!T7dGXt>7wttFsP@6a3hFagdOM?dt0O2ZD?tx7{uWRHQ>l&GHnd(|O; z37q^Cb>aZ<{C7TPAyN)ZwE;(60kI(IqH{fA7H-<>5CU%!80sVG%zW>!xLG!?&wne_ zK|nWf01{36nh2K6kRI6l-7md91FzucpU!h(|46|9c`l?{Ar{QqNn^g*mG2*cQ$##} z;I|BgAsp=q%nQO_%@W~#J*MJe^`xPgRB{<}$n#%@`R%f_6LD<@5nkc01`(Ozg{&6S z17>#q-gohqQxjLKBzq%hG+=YhV?w^7MB@hO(AZf-p95bp=)5{QI>`Pd?Ofyd9(r0Ws*KS{6K)YgTf3^*2s7a zaANb&XG(l4ZY7@kIo3~U%Rv0e%}^lURQMu}GI5El{u=yxi{eAC_bE&tQ&!z?&uEQ# z|JFUKgUQ(nUA;l6J<+nb$Ybia{SOF=B}UbbK!VbT2X(OpFheGfI>rhke)DkUI&h&x zsTzdP#?%3P$hg&2;0pZ?M1cVU$*pm_#38ED6sj(15K0;huj%RSP$}O2MjoIsjQ&8u z#%+Qah|*beXgbk7A8sLU6BpG$_1fF_{5!7er-!8psYM>;t{&&YEoP#u*t&klTkt48 zDZZge^s3)jC2uq^Q7{{46M(utCiIFsLsM7i_~L}ebJlbvi}h>obxrvgC%>AA+6NG` zcpCI;JiK|z@5^<)igfRFX+s3if3WhwfQ-oR?)-k(t|%(*dxg5fkZ__i^~5_qcE;I* zq6hbjfX>NAKPE#P1VVcL>E>4M+9zW34Fnr4XsL)`J6?Im{={B9 z4W5LnG49T&$<|2A6puI0lB?7-f2D}>){3uj#b8<4WcaPa{8ZpSFT)XFYSZ;vFES=I86YxErLNcDj$>++yt0n5d@~UQd3&DTE`E6 z*AaX!Xib$aCr6;^+_ltZv@-3@-&_Go10=mvjVt|T_ZyQL@rT?j=?N)9g}-{ejVMrB zw?Fcj}3A4mnJ~MCUx#P_jn@v z-4fyb*3i<5mVGe3e;}Ms9Uc*J?ZypW@!jz%7M#_vCLUiYnLT zE^gTG<>ShL1&o>0FFqPGAk5}-)A+#wEI~x(6zwB1KWM_}g zjY@ZrQt1u6wytk}nqBGWbQNa^i`-I{gAN>j*(v7;!feH3y1Uhe47t7CVlC(w*Vns1 zdg=Hz3s%6?)YO_CdNww?64?*mZDq*#ds^YolqF83T+G4Fjr=pnon2YNcD z&!}*#v@ZmVIiY2m>i^;6BOxI-ceT{DEf!k_rIzP<*j6(q;O=p!Eha9W5&zNF&s&#d zYnW727T=Mu*!!LHI~=d<^|yVy+kE|z_*>1@mZ<}bI&W-jY~+Ffc<<^{x*rMOycrL; z9POEj>ljl9lRsi7F%$wzQITJXbsP{ILl4hgIw8B(^PzpL4mJ{yWFm38-Q1o_MIm+f z{|6cVf9p1;Mff@}LuR=xayS~@Qy0u61EBcV1KV!-y^1B^W{DVc7iApGQ5B;`8@Kkf zE_@O0m?LPskj-*;iQ*kLR0KcKK%Gdqg6(welOXsyHHN9BUuKLwDH;JDG#6LuqJg-; zhM%ThD|Zu6@x`iwC|{N#l(zhdSgO-Rlnf}41n%wBPjQHzW?yIDXq$<1`%$(d#>cXL zGD#)xBVNI--?})s{yIV?G?SDk^;)(ckyO3hf`(^#1u1E2_sG{>T=3%LgO`(NSCicn z8S1lhV`{(Ur)DJM$0vtx3~*i}lln_i0BAg{0T*qYm=k?CaWH_On9*pxUo>-v#eCUYZ?L=~M3;wdAh2omjT5tK@w*B4wW?>Z%VdYr+GK2d z==zLT3D6?oxH95VF_BGzb-G8XMB|F@+~J9XvCD8??433 z^IrJHn1Y;joErA!KoX0a$Kv-u-Ki4A#hDXkD{D4hJN%qpv}h$!n4^(tV)2v|Zrz&f z_OrfZrG~*9WVxTDqpq=r=j_$`{C4`zE%lxjS4 z3y3sukKWM$*lvY;skOuKhEC;zFrf}(Imx#m2#|A+LbGPuTUM|Ja_k-Ajd@``3kz3* zF9m87ew9^3{W$N(a`5}j$PQuNhncrTpoue#-`-EVrw$w8rB^*sTK#EhX^_WI=Yeo9kfXymTZcXst;_zMf!BVlLnJKqS@I5=-T%R><8od^B^^uuOeCGPe z4c(gwNXH19-xg)E_GjEC!azUF9~Fo@{Zj}uUMr3*LliahKY?XG`ZK<=JAWb!qj>i;w?K8z!2YKd3cSGZl@ZE1 zepCNu7F{3B{YKk?c_cL9%Kk56oqi&-3rFTg-zZ{y11_0j+H;=t2T53G)A5-S4=vk6 z4;`Ju!NZ1AJQvrkq1&J0#LRa%XKbfWt-=XnZxY1HU63RGtsk(hkFc~)-@td9bBjPC zBJ3&ZIR^aE3&wy()4XDJX;lo}CkeIk8h8t+V_jLpQz)5L*>4r#v3L&NbBw+0t0;&w z%!$z$4-D4g#Cp^ctR;~~iO0lOXnL|{=& zr50gmkvT&;=001yO1fkVipi=?$&jsRjv^@!{*x*?b;px2DW$>U3L%N>q#Q{RFFrjW zHj?1{zUtkk!%(BbYG@s=W5~fYis^fC_75$}*51Bz(-R(e361oa#(%kjDoi66P$+-^ zb1qtWLy^0_Z^!Bom`w^A7@Q}*IB9H4f0Rj$6t+1*&M<;*600usMDwT`N95$_fwyJG zl*O47VJfWZ!4nolWaT23SOfocnkHC~+UNc$8flL&8f8!Kt8Aa+UFBHj^jHH;?ANxp z7pMLu|99ls#+3&b_rpB;{k#ZFv|EKW#LWm?h3H`=pnyQ_povK?Anl6fAU_ZY)y|ngAl?lw*I3 z4`uK&tffT(+)$T@FZR{uSoO%MIR{)~cV#5bc(fo)5NB(Ef00l2MQ`m;67AQqg^+Fk z7dMvfXEsS?@#SDo7-QE?x0%T^=ecMx_7ms)2PqV;*4mg62?jrL9`je1zD=NIIi~T# zt7vl}N%fiomn@F6c6!9n!ACoZ)db{MMWL^`{T?^7y2i(0)^M(6rfb%#inPU&wOC>z zjEo{2#_;l=f7!k~eEqWik(Aqb%1`%e&;L4Gb3LxXr%+s4#l#VwqGIoOM!kEoBb&0- zMsId>qA#TRb8ZTTcjpr$hijsZz^$d7sd_}kxLXPwI~dqPEr|1}VSH0sw=B0~0t4zs zh70Ic)G?4ufI<&n)z5>2&lh#hfZq_fzr5Wpr_vHTC(`&0$6LIHRT0ih3 z8_+ZwdV>hfV=X)8&5!#WwfA`7ZPDj11B)fdMxH~WQc`>; zXZ2rfz!G^@IZ;(Wzu4G64{{U#L8$!JBYEBHP<;@Yc?b_tg8o9-8K@=bSU%>-Oyq5) z_SH7=Aq#$TPU?3K36)AIn~KWXxuC&P4(lo$)!oPUKdLAtJQgqJV5^vaEW!ZkvYwL1 zzH9)@{9n9<-QecbtRxvgE<6;7uCA`S74{k;rWFsE#F}&pb$xm_GuX&el2Wt>L8~ph zL<}*MInqVnYYE2eSHo+<=dd_2e%eQ z@vP~WWO|QKm&B?n!<%Zqme%2YE4J`dx5vsWcRxuZ5YI4%&(j@|F~P@)TqiHX@O2sq$|g^6SW;m(7122MY{ zSf{<01&kYg8DI%QHztCYI%pZd?2D5-y{Z*v8-W-5lQgP_4W79L4FY?55!`;AkoBY_DkcRDa z8GxK5pVTf;#^mdbhk~{k74bB?gd2SY$8*Zbj}>^4$nnt-+c6~ws2!F-G|vKEtpI^3 z_&UG8-VuswJiq~nMhp=ZqaYQt#k0!tdGkY$Rv2iP6}YlOIwhP?Q)AFXoX4z#VB6>I zoLw7i@IT2m#^Xg2_nS6YE7o&gGPYPXoU#~p+pXNK5S*1i_A_`Fw6yee$OVDvh-s|e ziwX_`1g3xcmTY|tM1@11UJ-lN!2I@@8Qqkvv8ib%QW9=m0$2=*N-Nubyhfsk09`0I zh&ID(2M2HrBNq}K1#`lss#2IitoqYi`qCsUUK75{x*0fRl4prkvy`w+_giOXtRxhv zB_U(=J0E6RJ5ln$W;QS2lsJd!vaE1aW|x@*wwa^CVbE}oy6^klY(=*38AOR$zADAM zdHCcbJx#&-4}cc)I4@+$A@HhNV|8}_&uPf5d#%s~_1D?~#^=!s1$VPl;5oLppG7_r z@6AQ%6m**JB(*Lv9Gxr&8(;OtOzVdm&>1bFkvTCWwaOH!a^rQLOk{UJ2 zsa0)$+KxF;rX5&%N(!70Jcz~Bd=2(@!VC2X(|dVVR>%Mkn?Is3SnWI0CM+x>52X8(7y7io8Dx%UT44Olcaz_ z2fKlCZJ!~F=f+%zs3v0j7FPpv5J{Un#o5n~MCchAK?>#<5^^Z1B7MD&_cprC@@^~d zRjL47ln9AIn1X?9+jE~}&QOtwF2Q@eqOl5+>h^2snNG7?X8g3*Bqe!vz9p_7I^B$F zG~)MGPW=2X%&yDh@Xfm@DA*VZbHELstu-bC_cg$9t9nT}m$tQ&@hK^B_&t!A!(IKc zj!}@mc7lNn_ff9QieJeI3cPh{_^f869@K1?){bKG^$GTscBd5g-1~Y zmVJF+5T3)nw-s_fa`_t!og`#r(Ia{#dRI$T^9<$x7@iBhG`$r+r)F>Hw$Nh=2l+umkf5NK&Pa?WJ z@2XD}MY{Cjy@lWSR}9SOo$N=Rc#QexKAhRTHo3*&_J{TRpMNf$&FX6;LR|4hxqBM1$W~yz>=Um|77CU}yso+q+Y4Cj0^d@PqF)4g}+RYMz{- z>px(D7^;Liak~47Gc%Ed!!$J${*W=GB~aPMg7XL4%fgyE{L^2*ejVMrF#B>XH)WY1 zxtE@J>dMiR8~qtO69Q-5Nh!XVDHKIJ+SvCj5~>_C{k|lz`@@ZZaKfz(Y(C(}0)vWx zi^;)FL=>|@+$D$)L#WFMk~0ANa9)6_AoP-0#AM3^bkpYG>A-Y`Y`4>6tKic~*&VLk z_7?^hnUc=75OB6!W`FU|lt^K;K&qPEudb^7yoIOJof;xfML!hsy;Ogc$S62-`QoL> zuuIS71yRlev7g8FvFjNTpB>K>GwOu=IHz~5F)P(jS8m3@c-LpGcDKey5S|E7dL^x$%++MOU%A}oTa8Uf9&2d=QSeselP?t&UL(9;ZR(Z1nrr30?Bun zQcFq%Ak7&$(pI{ht5>9a!}9TCg82E!Y3?dJ)$RE|e?;asY7FXO|6TO~@dEBap@`{b z&f>{0Z)jlo`Q5*?K-H`cl@0UeXmlnnN4rqzT8vR0aj?8?>5ZxjV@s>DvSVF46J#PZ zE#7&{O1u7pd@a+p_}3FDr9#imj#Xvv&UsY|ypHm!b%_?1Hy^_*v0!h0TX*J;mvd$C z4&`uz&DgWb^7?w_DQRt=*ZwDvc|+t?*@t|udhV%jFDRgSOob{8ORSIKa;m6bx`m(W zFsmvzJ_WEU%Gm={;dok&xxZ2O@neYD@h|<{je~QhMHD5g%C0@jFaDv?9W=V6IH2%& zN$1;|$;%4`)VHwlFOH2I8n~ohLOv!ePe;gn^Q(~Zd||oADhZ2a@dk~GVayAzYsG`{ zYpKv39mx{C9b+-DnR{acB$-H>4y)RLMWAGEPonmKuql$PpkvlY%t;yVc4!? zhMIDTrr?h*Mvi1Wiw(Ztms}k;=9GtS0M4#>P^4E><2W1a*|_2`S^vjxZ(H+wCHK~j zxc$vfMP|*y0z;lsi*}(jEuWo6`#roA#F_V>N`1qHwK*AdNa5AZK`AyW4bQWrT8 zVj{H)uGh4xh7qUlWaldE3ns6V6~AZtVQ{MmFP_VskL}$bBJr0Y#?2L@LsWzz!i!$G zPxck;EZ-Kw8x#7%iCDbx>+E0LjX#-jF(w!^6|V6Nl5VCu6}#x|_^>dUaj|mKFCaT^ zOyTF36B&9}nFAHOhi`^-zI@$%vwjGE-vhR#L#YfcaP+kalZ0AGj;AvnG~j>4`zgOF zjo(vCD0=akqu6}5i~XtaQkTI65^w&)L%Rcao005Iy%{3nqn45AZ(0}j4HYuUn0HBA?WW`=AB8F8oKfZtgX*!IDSM8@+mZgao?6rH7?2?2I{*M<`!&~cG#anV$-GnNwpLa!<*PJ& zg@Q9>_2=a|lDcO=W6iy5@lVk|upZzsL%!MK$rBY@GB7YgJ?wPRZg^CC`~CJ0NtXL} zyi963xU6a5U3^QH8AN#}GA`d4;~0G>P_Mj%GMTE2l2Ih3-@MUcnD{{-9yUVcz- zz|D(xz}~Dqz-+>A9|Z3g$4hz3IiBLJ*OLb$rOid9_)?)Mj0(=!hq*ol*$+bXF@fPh z?zR_4+Kb0=4g9MQCo%GZD<*ZIv6|az*z-$gkipqGdw6qhLlQ|bqP=U(0(Rx3)4CtE zFRw!+8rmd)PSxcbDU$ZQvRloyi~VmeK$9sxJt0Y3;3(q#mIU+dc;jUfMWF8B%xM8# zFYSB8%59Zmy{%)EbP?wF@icb>S(ap|lD0kRZ&@~@tBBAH131{8a&5F=+!-CEZ53AH zmk8V2v!H)BI5~P=%jw*gRaxOTlOF1>q=9;l?)HlNWM?>nwEvtn)>)(KNG%Unb6|P7 zb4^Mr8hKFuFCD?;4m!P|4U`(A5HocKZg(B4`tf_{;|DMcv>>`<1C)(K+gCIBS*C=~ z@P{eNCtS|ZQoZoV*;ej}>Ct*_nEytR$B=--CDw#l=!f3h9|#bng^&!QjgL=Fh0)x7 z_-jL7*!c%9+H>kFWBLS{F;;;;qb&$`dLErTNmyrMpi=!Is@dKW4Bcf-C!Y+^RN;RO z2_SQ}i9V#4_NUdkTYqH-oXt8BV}&_WcJirC$vemTuEP=R+vie~jg(3&iLYBDlILDb z)7}NdSWe3Z^aMOH;7!{9dW_y_oIV{<&p%Tl)}JXG1B032mct7mG~wjGl&bMT?T_qj z&Wn=J8w~E{cFUEuwOqLA2LGu!JiwQa zXQA>5V$6$w1by@wN=tYA@$W0*Lba)ftIUpAO+Jicbay3R7 zv?+$W9DYA;a29BNE-{n#pb)*#5Rva*MiE;qm>zs{e=)Q;*)`91Z_cI`DQ%WcP7 zZr8>RC~C{2l`aRt(`JNBKh>cf%x-mcb=IPcfMO*BPU;sG)nTqPc<=xZ)!W-U1E(a7 zG^dU_;)_m8YZ&l(BElf9))?lW_Xx}bxPdG>n`gD9F|egl!?FcZ&)uGuD1MF3=EXP* z)5sLxkqT3x>qVD=UUmt-YC=~-mX%^Uig0$2z7wEvoB{G2oG(WiVTpJ3_mjcf<9kH> zC!v9mLvYih+I5j*@!O0S3w?Ol*6c^vBdMA0uz3Fdclll;K8CSFeTPf*p>tg5Q!9o% zf6A9TNaQ$OF2Hol;k6$a_TX0n0txu8|5^tQ;%V06X-HNc@vgJwuDMEql(6Vt7uhwR zOMYdD?Y`dtd$y{@nWIEZ_U7) z;O}>r{z|Xs`FAsEoKN@}`f6e8)#c-w;M`weUJOVx^Iye_H8}G`Mqmd8YeEH!mFOKe zFP+M4)@YU(CMHR(RDF6OhasU;3=Sa)v%}r6r*GiJ$_3xsK-m^9NLQu=nz!5W#n@an z1FGWHaFZuui(`&2#GIg5ac+BEZ0Xtx*6KL@Di=u`(Wnsq6Af`9Mzm7$0O|0rj*GT0 ze2=^0HTP5)gHkX0B((o0k%zXC-3i6WAEz78?F4aJ<i zQzF#sFI_dW@E6eyKBJ@Mno2TNV6@T1&g)<0W_v$m@03T@u-62xS7*BA<-7BY6t z+lMVTTJ1@tw(v^s^V4pe9InTd%7V4^%VP*1sx+9=#n7F_tE#T{*)$1TEH();GNtS> zoA{VB**{%rljC(L+x+4xQOa1D9y^J(M%}LYFADYdN1_y#y_jG;f)Y}QFyq5MZ@J&I z>jw=K)+MuEtb!(=LV+7GEC3f!i7Z>d(KYSDc*^zydA`2$Rwv zK43I-X+6{Z=|0%F%dnCm?Yyg2aOS5pO7v(!@RwNw*GnwkC2JZy>(^4vu`W!7A@`|22wg-4`7KuW96FS_t_~iaK``%(;}SSm2wK2bNJv%hh(68uwL3;J%fX!%i}ViO+au zc5zCd{jX*1p3GdYMLP2H&es$&6luOFSoqZQUl{O|xak@!4XW0a$M&!J5fg^$H4t!Z z*#bd8>HleG*)v0nNWaXK<#yt~mBA+8Lv@6RPsHVmOf4QSq8`SjJ!k1PO+N`=g&I+6 z*>b<%YEkO>m%YsoXJk-;FuM%3xg%e`wzTI;_pZ|=s_^mYX%+BQG}sR@lN(Eh5p3K* zO~nA^AuJ~+Bo%(V|1SAft+7gp9@Fi{(DSonqmTDual;6Uv&PE@*DEG{8Yc(>BKJ*YQFna7oZJ zk~?;bHw-Qof^J_~<4X)>!%Ip;?W^w6-jTP}!SNwAWm9oy07GQM2hI+@BH@2OxM>9( zYujJwA0-_^ubhhGpWn|8wKbs8{C)g?8mY^HH+T*pSP=aB0yhT^=b?e4V_tS|L9ZI~ zIO&IU?<%YoG4K3!A>F+O`^lP&{QJha0V0G&v@( z{CKU($inp;dp;~Lg>1}CbohHP7z$_RxFx@0iX#M_-P7-DlY7Yfx;!t6vvW56AveM_ zU(B{#Ip1URk)~H$hp+X3T1{*UD+hK>Latkk8B;?kWQWiP+7AUSv!#KrK3>4yH(OVs|ew$E>P40tYtUe@=839dr%*|BI1rAiF~krSF%dh=8OD z0ly+x86jUy2xBe(^xB#rSgk3;&!dlSK8nu_v-m7_tKs0vN?#ecI^i;*c%&A{?zYKT z2P7bY%LH@`mo^~sCa(0cWnTi}D!^CF+(VNAqj!8n|B2Q1s019B1bl0d3; z!871Blz=@S=|BUnK1Dsse{|ul^MBzK2qdS2xQ&QoXwbpo*Fu7*fa4MP<8QwVw;c^x z#j*OVTtjk+xesf>mabhG`bZ>7r$B2KlFpXsiB)kV76YMK26CW0>>tzg_uJ9G3v2)8#JMYbh4wCiV0tUd-nLz+3Uk_$)nM#8-J#u5c@W3c z^6h^GLG(5C-B2DH_zXX|FO&^I+8DqkxO*!MGX70&-PsX`hOFFr%Qp+=cU?}~8Nwmf znN!&bPG=~fQbR{9fCC12V>mFu8xb#iGX~v#4L#B4B0HJUL{zX0smj@XTo&DNeF1FG zaKyl0-`WyI5EoiZ)g})156CKAB2i>cmyPthK5RLr#T$0$p*Q$%HL6YNU~w}N_pYN; zdzoUjF6&mUok1Zb)IH%}6}wy=@sxJPc0iFx@YhDjxSDKKb8->glNB!wPbOi~IJg=H z@#p7m9>LbE1bZsv&LLJ0CVFlJup^Q&NTnN|#w%3STdI0k7vjbHZ?y{-9mNyVnzxcp zmABNwo0oHI{ULd20eB1+`@h!LkmPAq72)-l?c?Z=^s;=_yeliE*=T>NAcUju1Vde7 zPq$x|v%f#9nePTc#n;<$t6f`5S*@|>{ET!70%?}T)Qai)<#R$Z&j#i$4 z3Q+a@MJgi@i#xx6wCW)1G9K#yLI^$!K+#HA6eI11Co@6|OirdR<_t_sFv4z(MAH2a zo(|!+0z-vR0=kUoUV4#_IP=%eT(RiX$Tyrw{x$=D)tw0dEa*Lv9hRPRQQS~%a#}tD znZGC!5!K8PAR~N^4#YoQeTCO1@B7GXJy7^c#SobvPxXbe59dj0>>6VZ5#X~)-E4_w9op&3}tXolGVNd#Z53g)VLJvRx={lUah^+UkeO&NH>@= z`3@3>hS?6h7m!;K^g;+l7Op(vOuH}ZOBHP{CjKwWmlt(s z#;Pgl&#k=G%1f5v-NC3FogtI-c&=;0nWTo8;BzDNVA!g0%G>p-0Cpl>)<uw0)%B0pB_)0Ct*viV)ESsf2M)VE z*SO*Yu`LY^jmWr%C3B0;Gqu)DuZwyb@7ynhP=v+J znI^U0dWL0waj4b(^GCC?$0`((g{Stii)~WRaX9xq`dD+ey5=lPjdjO6bfy<;Ca=EG zqmc8RM*;@nt$(h|1D2TiKsqr#0F4p;Nr{j*L^8SZp@9tRkaJ2sLkma_g}ruRJgDpz`Rq-ZAn z9sLb!;J#(rpZws$)_TXuG4nygPpc!xO?fGaQ=Hh@L2H!cUViUoscO8bIYyifFlwdM?iZ~#^fj8 zLK~b#owX;z==;aa_3d`dg=LbKnGm3$*BLxlNB9~#Yd=f13POZQa_~TCaN5P5k`Gxn zuYdDCWgz64I-%IEWOAOp#pZ6ugtRpKZx@-dr)orZY|4 zwDtxJwEvEno;(vz+nZYVk1)_EeGA~T!AB9_)DC|{=;VPbNeX7|=aO(cScDLeSxpA(9xPooT$%*Vg%ssePX zo9Yw_*uOL6wJb^YxAh6O&^?HrU4VCXE~V!&@S;!_2ueV}{;L2=mI2rd;VHv@h!idE z{Ybj2WK#ni@xZ1zxV^x9hm0$u*wW=QgTI$C?@dp*htMRegecOITjObFEk&glw9bZQ zPZ>4MO|(#zkk3@hB}vx|-u!BDopCJSusVQ>1-jWO!`VsNn&Fu7-%P_sE+hk$&hn)a zW5<(k9>nTqhKB;8dZ^KqPpf3K$V&IUaKN_sOWb;NMs#S-t%8xLwPTNIFICs$UB3V! z2RxVvJvaL#I{JAlMuA`?4mK^R`A`!)_vC)D=s$*e_n&Spyg5H;SRc4^9FyxMQsJ|! z4l1G3vH-oFu_O6#*q=|RKwb|5+_O{i9^eukvnKV8lB+Q>_A45X?Aj5Na`ghf#~Eq$CWFMh0rNxTJ1h#2$s?efqf_ z6u%7q3nJQqb@3SwCVZ*e_Z)u_NN%^v&xJuoK6_tNZ(O;tLA2bn(olf!3z3DD6_K>G zbV#j`pWMXcphp^hzV=hc{q|in3_vxc(EOyylBWf2!^(J5k0DV*kJi3!X8!rUR-} zpuKx4~@2KW#Q58>rkg^IEJ zAw5Bj>*U4L>)2NzpCl^R#HN>qx=OH*;J$-7ul<&qZFRpHFPnRWUhDYsbwet3L#zgw zTTCH6MpBNmW8y@#LivWby-M)<@JW0JzB#-m;WKSYRQ;>Y=g~C3V~}mhyv+%YvCE8Q znAKd&s#|NH!NrYOW0?|~GXfhoR_t@PlQhR zQ}#p_IzMm8E!(qN(g%KEm$Y<{1G&w+ft99nqnD@m_2h89Vc2geF?sNx1fkK_hHP+1 z_2v^-`JY^!1XyFdIaAViYczPEFac@0GO?Z2N!s>1=jawTj`B{k`hwEtt3OhKqr#e0 zAZfygC>pRWVg2$kV%PvJx$}B2eQz@SxVLz!sT%CSIjV#+rZ{U|jcK^sNKXi5b=Lm} zcW)gQ)!Og>58W_?bPe4lCGCK;f^;{Ev~-uGbfX9oB8rj{(lLa9l+qm%N_WHW9`||9 z^L+n2*ZKW*-21xrh3uJGv(~-t^@;cA^}g{FptopKP3(XMP(Q8*WZi{qES0R7IJ+C% zZOlCX0n;AJ`y6Mp2tsxqnq*L2)lo zwjq(XAI_FaEIqmFAu$4T5_+Woexg&GsV#hyty+}SzuM`p1w{vJX22>J&ILjQGhlxT z149^P=HPfgQ-1#yZBSpXi@oaEI#|C?_T5L3&uoL)!w)Nnn$6JxeLAWCVa7=^uJH)H zIQWt>4|l~M9i5ozkRCelC#I!d`WjyNaTX2#_9&3o{yqJW^SI3ny1b70BCfb5(rE^i z7K|K#LS2Uq2ekMcJZ8L(EX=r4Nu1MUX@}I;EW6PcSoY~S7Ir*&)$#O%@DOR2f%!jQ zjzbpI5qEzU97^4(wRk4`+WU94NfRUX@U~Z>7_K265tgXRM)EJW`VDtffZ|1RcmV|% z*m3|)PYXmHuTbf2$LQ~gUnEWIc>(LB!eZ-|tUt2+mUZK3?=jcGK0D{{tx95VkppPg z&6>-Nq6cXMz?lK+8xfc_O@7<4lzAIvz>dH7z6YcU?g9-_f|+ z5^Gh7eV4o^bp3`#3-DiG%b9%&8yE_X7~ZzBvO)!e8NsZz?c+v;njDM3jVx^U(r%TZ zXXj`LY9xda`Y~{eyCpHeA16}J=XX)2>z`51O5{464?Hymjc4X`Rf4T&gAB)Pf*A^H z-wat!hkEbCmy$@F@-tNcK5nw06K8BKQlkA1LRF+zN>de2A2N(vERMx z9vSoN=AKQ06qYhw+!5SB%{{3hd9L*7Y3SkcFol;A<|?He|7 z{CNq9mb$kGOn1Oi0bC{I?C)PO+8ijAf=U%61mx>WQ%~RCq3cIpAP^{n3V`f+uZHVd zoXPc^eGRf;2oM6WoJT7lm+S#RJA)>$Z<;iE5wz@D+uP#;9rf#t`Dgz{=dv_Xek2?D zeGAX(xZ=RkW1_r+ySy!$HR+Xf)URL9D=7xUyfC(li1M5GOzyszVU~?@`4+h5Jj%c5 zu^E^Z*&T|8^;f(s4J}CP0B}5@fdl+8aKuwA=b|Ee^UX4nlCkT#{*N5ZLHQObH9I~& zwk!7sShY&|Q4qZ#Cz?U>1T@LWX5-`q(UHvQ36g$!C~Zv6I6?L~{(ktd3Emy&K*2=C8We=E>z|L>Sy>sEky?~oCbvft4W)daN3pAl&F=Gv zv62sc`kep8j|z+gQ%Z(^--;DTb$hCwldk6Ky$gh5IlopWRb_c4dKg?x+rnF80GjLR z+dw^!YtWuUqK!c37D&A|p6dQi{Lr!v9cFeKUfJKU;L0Aba&~@HJq)N2o7b1`BSz2g z6$x`PjN43T2IA-I`Ueb9&tI`CtQhW-S@rUuHb#KF0Td`=WO~VCyMX4{VusMFx0k0l z?O$2|e8nfZAykT8d6Ws+9aj0!EIAE4Fl*m$MT~0*@X^N$zQueJP?0q<|e1 zQl_R1ftyU@oIl>60B>i{lcYB?DA>{??@=kt0`JP*WI`83Olss0X9#j~C|9v!l#zJRIf_FGe@L=ABS1QQe$+Wehp70RJdf{w9UN@Ir2%bv7NDDgm!8juVBJ1l!87~H-k;dBEuh&n}Xr3!Ed+ zQU}jM&<90|Dmuew<*7gfbioUr(@5%_LFtDYMJW2nLdLiFUlqHVJR;e898qtIM)^q$2h7qYG=ai${?&4>!1G3ze_k>^mhegi+Fh~2p+JpRGbzV8U zK{Fo&`bc-4nOZ>o1K`8X&hhM6PiT z*`DcVhqUAl88O@GFOGq#=KJAQH)1cEkpKX)1Ccq{2}<=|?4BNEu80HC&mB{LuJpcn z1JwIr>Jg%#T^(~^Y;2uxDQx-PJlOHL0 z%s(pyUI~T1Jy%zkKrUr#_?R%R>sPHVD7yh1AQ%uFv$u3`zynH#>KlOTQso3+8UO)C zz%VlfsO%Xz01hsAE&?yIx*6~S5ToMD@6#G^rT^`MDgZSLpb^bUiex{uE48$X$kPs8 zn2gl9|2crBZ+l_^b~VoT{p+j&=C+W|A_|0u&SI3GQ2yS=fO(W2)3eAuuN*Wba+nJ`=iUWbwAZEy)>vN58(;21}Ly&;fN=$7vwobpZ9=%kh8QUcewK@XrEk@2?J>l7{Am zB_-(Lb*#NN86-eJYn`k?e5}g(w-9p3fsgo*ulG@B4@_mnCfK$X zn?E!Esa4uz$2_b#$W*`tvU+?%fiu^o{cs8p9*NL!jnKgT4F;OZ7G-o~JvKlnn3ooY z;Wzqx)W{luS61B&J#K%vq;M1=9c93Tcim80RcG7r2Vx`AB#Sje(N<6G zAGEeatX)Zl!1VdUACYPTK?D}SGarpBtoyJzD)`s%w_cF|!Zy{nrb8)>qfohv6=W-N z6G1rP3FHM>#zMh%Jm>EYPPMPOi!ma`+scevRqlQ88JWR3croR# zHO??PT{z-gR~TmUS_^l~-ZR7@F)B~`>(%F3(X<)*$>x-i;~O;|**@Qi{vpQ{h#A^!uY6$!U*@fld+^A+RX^Xw=}&T-uKGzVrtlu*0Z|cIFrcG=S=hGa^D?3Zx>ZF+ggj)k`5=Y7$B2HV3j`e4$}4Ln-6YWiHWo6*vJ{LnwfP_<9XgwJra?~Vf;GH5{} zD+J_2wN7Ab1R`G0o%^f!ECKRh>!-yNl!DecHt(NWX2CV`>ZlMgYPY$z?gI zc7vjWO%BW#0|*b8i3G8o+iB6drL{F0vL_r0g2T#*-AgP?@Z18E_ZfrCB7TraR`2aM zekeh8IC6+b1B}t@`4MalDDmn~?_{_Y7VG_%q^HgLOP4v!+-v}J1@PF5yrbl3!+@{0 zqwBfq1+Dgr{{`Y^z)uAg1<-$&!0%mk%gV}v_M2ZLcFdAp7&EfNUw!XyzJmJf(MX2u ztd@fiYow<5Ru16ZL2?ALC@T%w6C%P3m;COkQ~_;YWtoJHiMo8m{Lj|?N-9dhdW}{& z?z5WG`I)%reA^u*yKkk^&z+cy`vi!=MD&PQ!i|nQFLJOP35d}U6`@A9O2FAuXM2G> ztckd5L^lYIk-x3{tiuIDI#6B!%}Yu;x;wFBvRhuXSYUl1X-TD2bUiWiP;ln{!hlTx z`R?(dBw!1UFMvs3y-&xly&~)Df_eil6fXI$^eRtBtDeDRys!m$+kHf@n#G<2*{Eh)kz?kxoUCvwr^z zsG#Iq7Ld|JIrVLO^P*GkdlUcU15}eh|JWO!3Ke)FOJ+ap)fOG(9ebk1N%3vAU4EMRK{psO z-}eR#1(3H}?e8qKCYs+~On?%!a)P?`$=EHnE_-#LU~5I=S1KYZCbpjlz>k`$mcNFd z(TZP6d=CF^8HX$7XdXyN5fAIKrak5 zzQ}(O`Y5HCd!}aaO0Zu3{hY0^%dI5-9V7-E2|QE48v}#=CSP4q zKwH(>nUKF}FKkkA#?urj0YEy8S#~Ae^CSF-=5AHqjkQhk$NUDfZ<`Tl6_B8k2Vzkm zGWuKOE+Rt`!t~7pMM{`iM30eAEbs#c0b}4V9xbf5Wd4kD=Y)SIg5&dQ@PH0}r}8b6 zjGLSzPx{1zS3F_DUb*6CQX!fbKUKSO`UOA`@1@m`ik5*8^m?5ZSLh z_P!-e1`R`SHGvXX>hF|wZU87fgZvEiCI1$;P9GtW;XpnNN}dG{?03L^X42q+1NdhG z)+#oMZoXyJk08)Rm{C`DF{aE`s9=sbR9pxfEUZqT{{!6!Na+QZ0^ zb`WNQh1qm6?LbMB`l%7fZjkH}P-H~{S&C(^KZ5ebKiXm?hCuWpvJ-~ssGhfhwEOR7 z7g*{KM(*gm)CEC;`}*M0#z>)+^>0x30&g&CFJj|HC3W7BN|~MJ56!E1+*!)+++Jha z*^|g9dwO@IRpp1#6=d*4s#*^S%1NfIWr=fE;%QnVjKRZ7-IDc?luSFZfBPdsG}rlt z(E3IcS;9cObEKs6w}CHp_O9+(SMs-q(c~9;LTG51_&*h-Boh>z;eZA=U9tRe!#g1{ z(K^7SjC9_C1_mWiRjqSC#IYuU*Ndur))D2>Q&LhY-1Sns(|Ijl zhqawu<~fQ5CD+t$EA|IcPsZLmiGsH~G3Rf`K7^ z4dt!QP0+y+Fg-iVcD;@-wlt{tQCRWA4Wl*v`}Xzz_Ppgb_?=zqyIX~?mXrYPhcrZU zN>R|rqrv=cL$2qizO{`-_f_Ege2p3l2@*L&Z~@vX*WSSF>hBAGZ5E#sfW`0`B-g2( z(X1AoBoKVc_dsV&k%QH|Io`-0z>G0N8DXv3^;7wG`2thi!Cj&)`V#4o`KQ_M+@1L{ zW8E}`UUi4t9}$CTe=V_>wpt^BcHODXQk;`4zgHLM z>qDtDV#nLBr)eN*#8FQ19ksufISoc=X0AQcgRXqiWBy$j_`f4B{$Hi`|2t7NQrTP^ z&=}f1tsi0;-Q{X8IXo;!nP+SyAGpn3Y0&4sD5?g5On3sXj}QU;uH`$p7}Yb|QOf|k zTImCn5#Z?t5k@VJ@9su+@K`eRU|W4pTujnCfVxOgQPJYHP~enTBY*;5Xg2vZ^ND7F zI&tng3esx`>EVZ*ZoU)y0I3QFE$9#Bb0O*9zL_+(y)c~%8^;Y%EORHCj=PT_iFwLJ z!Re0`Ij&x^Knf&sw>N>6=a|5N#*_aoM8b7(iw~ii7~OSfpFPYKQYp5M9I(8&eA#7{ zrUAgQ08z&Wm|mtJE!ue~M^Ijif_NEIT54HG2t>BXdPr3jQ^N^pv~Dz?n*y_Npi^cc z!AUl_548aO#+U256X5<7P=9q*6hP7U3}txO5>i+7a@eVNjI#xeg8`ECSY!)tw?ohE z&{@X`@dG9G&n=w54ICrLz$z(0!5FlI`dWb&8tfES`(9pNVB`a!62N37=te&10jyrd zbL6BGXj}0B31<8s7eGA(iWek`%mpa#$~I5zzdphOQqdQ%uijv)7CB!jzqPfs2)?Hk zuxGhnsyFlLI`C_e|FA0GQ}i6Jw25-*fUiL3-1%!KJx>?aqx@#@vT<^q2PSN@{L093 zb_Tu8?@V&nwz?aXAAvDttbzL4*ZxB(fNkcO2WlkXsN`M>1V6rgo~A&I_`u)%qi1O_ zp%zy<^6Y>>u5uF?bkJ{mW*@V8jpIeL+mxA7a(K+lacDR@IX$R+F!(({)(8zGYq&HR z9y4yo@Mpzw0vo1{d8VP-CgGeO&cm^f|K!lv#pmgUX`7JcEacA^x32VcS2r?IbAOaj zE+Mr03SzT`7J)Kn>yp{lVcCt!&Hx#x9>1y@|$;qtoH%M_nXLu zG5aMzMInq9Q--g4CO7(=xkk(M#FE95m ztf%aR|0cSolBFV`VGlhL&6zupsO4)#K?roM-M6R9dU+Yei?Q&Y!@=X>EVF zbo#~^EMcIz;nIgq|9CN}MH+Y@{hwED+&)uo~{#uA>+!c zqM9(bY5$2qrS+Q!Q@(j)w>|SYN_H|mK)Hd3UCLsaak>Jq8INqWD!Cno1)KjkjHzg6*eZ6jT=b$_$bpNvs0 z2WC!Z#*=smu3Y|j-K=CVnP#UJMhYWJ%7@+cXjILaPpAH;bEosy&HgT|?RUmMQF@q1 zP8XxsLM+-MP!iMs@Pc%%>3H1u(r+G^?{RzXTwO`EfEcnvATDsu1+gUXE)AqiA6ODj zw_jTY&H-H^fE)-&NHmmh)>bxwqB+v&@shfq5gYWq0d)=t^fFtHiUQ+TyB|uaassi{ z*5$qfbH?;((%S&MuBq5dpQ{6I&pfw7gT^EMOKP>!067P22+l3$qtDyx2g2&BRq$q| zRX3mpL=Kk|{hH+k{96#p0+ROBgrCF88W0eMgDBL~JhfEyV_(fW*tkHyEhA)kU?(-7b!tuX!?}0OrY5Wt)_v1ADP`{_)~K1! zPq=IA_UWedcG2iBzVA6bZ79%jmc3gOyVD_Q$8S1aXRES}(8ACRv`h)#x#}>)#)jpk z6Y?AO(2ZT$Do zi{!sLW3@`5^XV+^+=|YeZ9`7cAlR^A=p8)niJ1@XV$?5Bzl>NH4ju=#x>uJOi}!hU*J>pvS178(xaeok z>q6iPwuA*%OXqv0&MAy%4F6;5?;rExy-4)M~< z301qrtf67~IF~dpw=kfI_WV%9shwWGPS_^%j8Y+KUU_D7zVD1ih=c#g@))aT0 ztWrf*?xtHp(bLQ7FMXBTyHOr+Vw4_-_N2`0?9|P5&xu3{X3!LjY_t$ExJ`O`c`XB{ zB*}_Zr=8a=3$|{D2c0OwjCQ5m>`Pe9O)uhD>*(s{w*LSc1-}MHp4${PJ-wp5JR$?{ zwTLCZ3F$;x5)sQXlmlRp1%LaQdZxFhoNQV^je(A+0pUy?4Dc{O)86rl993TevC#y zSSTZtUmKX^#s!!oE5>NMTfRqugsnBtEpJA(H3Xpw3PjO>Qy6fy!tfONTOg%}pXK+k z;LK%!GJpwRkMk1+4@Q9K8k1}{z;Y^*CMwYhn_Zt*S5ScDsj=ow4=Z>+7TI*upk76R zR|!BzZnVv8bK4|7d$@3FQkRu4UO3jX6rg=bsd*CU{!LH?Zj2ZHY-giZkArOQqedWZ zPa=ctFrSlx&fDs7Aj0t*i?QPEFBwvZNoqS$IjxSoyuq-E2(dXeXOf7HMZi(!Yq(N> zs;;;Wm}~Lk6rftzl3}Z=r$9&AUESq^KKjw=J}pLQ?U*cei7=QqR)j@>wp$o_ zk>0J-fX+Ypnu8cMsp-|BrbSA4t(}8|JX7+d5bWLVrO#@*{vI%I6_Nwr7ns;p!AN5A znwtX`)eYXGKoLx7A!(x1uvT5v-?RS!w}1vs100Bd`jK%o&l8K z4m@ZI9!cz7Gefv2?1XtiDK(K~$_n_(m7ZT|ii8!{V$~C&BiEl&P{JK!45Gz`XA zB11$0)s8kYGE(5~-7mIGs-+@V_6Tk$Ft`7{sjyi|j0 z@Q;eZX<~&_78)9KWMj#YlN-I$D0KN?u=kHhlp$lm<+q9uA4$edZj~SN7KNP7mavuW z4=IM+q$7cik0|8SX5nf-o(K>kPHeF4kWA34jl>%c2}wyRd?r6eY}oL~NGQY25`$uJ zaBwpw%3$GB3nB&#@mp+SX#sr_lJQ087AZUIyS&G4dXn4<6lxENa)lLO6*~)sL^8|0 ztKa$ael0xy?zP|~+olbZpR2IK!#BSZVUAyXuLT5Tj={x$X_=Qj(Le&6ppvwZ)6i3n zUOh=-3JO?A2oyg4V(Qx~mLLF3r>!*j9JINviLL04H+mqX!>1p3V8c{^#ss&VL?{to z&+on%8w>op#)1O5i*7*NW{QSNF}ziyq;UG>ky}WlS>868h0ecDly2VAyc+W*ciRD_rgCN~t78Up!3!323&jv0tfVbYA}oU*>DiZM9b;3dWdr0D zuTP2hq_Uz3vz%0`*?6UX&^>rsNbu7l?AI`@+bfp*IUs~a;<(-JVZMka)L|jHucn4a z+hZwE`;)pvGh}c*B*a^cbOa@Zgo!!=KZ?kFH`(-$2|8=r)0A-Vrb(;0PLW*(j>{O- z^;QsBiyUe@A@8lWt|9-9-xV8WNPLEBkz~TS< z&;Rr={*QcgN-tFEY~FXB4(WB)vU`{E-zU(st=0)SCU58DWGiq&oXSEJDtrA?whPp2 zo`DJwSSjRgQ~-LZZdWRT*s}lz+MHc-|Gr>j#0UY#a0^D{f&BXz6?8JiXUYHTs?G^6 zoaU#6K~Vu2U$r+6ymmw$Je>b)F*R5t;fP3oKuwNt^IPAK3%Jhm^ttx2E!*Y?sd zgbtYzO?r86!|vsBGIPAR^l+R{=M&dIaay=8&Bx?2x93BJ`S~bX^XDADjR5Nr7XD$= zW=4_rgam=mdv8Dv{t=s2q6L4pTG zz!IK05+MocdA@UY7Zk+-W;vyo#j!Dn)Cq`5TXuo&Z_Mch;Q0ZZEx7CUeVUy6!!{d` z#`oIVr0QRSvf=Lav_P~741+ek-h%8Luz+fvI;|>a5)i=N1HI?@9x%|{lk-#-RB@Zs zz^!7vTG#K{I~l|H^^%@PyFidi@u@c1_QS`WOrk79b^jE(cJK)Deq}^G$TjV=dR`P_ zGJ=p~j#VyC->WQ6SoCpW#(zQKd@P3z#yyZTT~}K}H$2qT)ZiObP4NIm$nexuj53Q> zUFDm%Z0doYI-9q@i_S5E&0*iEw!Sh1B?Gb}Z022LMKc zi5LVJy;EQ!uCA`0^`Zs#jw6$k2B*!*gm>9g2rEt65t*&hF=G5fg}*SA6`bDCJGM_P z=~cgD5_kvcyZx`5IIv*iHj~@$t5j}xcILXEuQfsHp~Klwc>;{Qc6cUtvBo$D3U32) ztd-|1Br%df!A!|=fLlIC^pP~N=Uyj?rb_jgJ1t2JI*5%oTX;ZG=IG=Eq>>E@@@!6mr1ct)1E)G+`tFQ42uoUrahLR7WCVrq0AbJjsk|30D1DO1`ki$VG}qXe3=Mf zEtywSHbV&}2&${YYiep563}$--CAK2G8Bn)uL)6R^E03)o(M0G%H4HL{qIlCfufr7 z3cit^eyRKoUc0#T<4mZ^H?ck6_yRd$ zgm#bh+WMG5`qRWm7Q)3FZAt%Lc4^^osQCOu_`l`{7SD$}Ec9$UR;d zbH*;@q>4%?IV`}+#SY6WW8W0TM+4L;W9smqzC>?j zgVD5bulB6Bl_PgFelDK5p#&f8NMU9(pXDpfoOl}MYhqttUt>qWbz?Wr>%nfbj4H&+ zVA?A>*?p5aHKmT@E7q3WEf9xh_k|Rm2R6fml~KXplL0z+mJcx}~W*%?ryOT)dh@6%OIHKJ;NK zO14)=S<9)zGco64+{JJWjBtoOH)dCJJks3mFi7N9obxY)jnmqUSb z`qh!+_ScQyE>n0&;Bu;lDrC*W&n@ekFJG#h;i z-Kv>1m50s{v0+NXz7+NME;;>gxDxr4w}G^I<+gN z{b0Y^37+2t7R+E~9}vFiw}vH*1RIZGZ>xj_O8Dw2sz+>J+(C$lql`u8E9pM(aW7-2{IkVQX0;nZ|Ri{rVYc|?|h zN8g=^Cq6|#Qzs5~Af!ebwER5&N_Z{*eJA#cGZ*AsMH*3i!FzkFWb4&SQ!`7X4nD&%D5adWSi-`m+Ro(bnqNxwqkfcM^(KHfxKI7+o6R2+8`BJiO0&|;5?%) zdeDvUky3|*e{j_e&eebHszod5eu<4q{0ju@%@WL^76S08=u*6 znDTiUWise*CO%b_5h{Y<-J{SJGf^+iVAKrShL3ltzn4l|GU9Bb;ODjCG0nYIL9>f7 z@;OmPgO4|>R6m1Q|3)Qu=flnID5AM1<<3&G*)Y$o$_WI<+_CqXq#D8o!(S*#E(@%- z4-GAGM(DHa?O$|qk-+ep36{|M87^898L@q!`Mg;y`SRrf>~T*RcT{-la)b~3wsSk( zE_DyjUq6+iv~4uu>*SrkU35uAJbsEhU#jAfm>v+}l2C3TY)W5+y)k_d0EvmU z>v}-cR*h#tOip$YM^Zb%#Xa2?R^_nM{^8O2(?h~}We2hMm!cRWrpui-Uu$KdU520TLwmxpMzOC7IYN0tkdSMi0o?dGeAxfy(~PlD^D;60UCwtG3> z6@D6u%Jw9GFtObo|7(sZrAW3wqZPu>fu8BL(m2+*stlx!K)B^z?Mpm&LuDA$F1~_O zaF#aIno3huzC75xsDI&cDZ8Rh4NI?In9Yi`5n9>0CGX~9O$W)7iGa^!*CpnrzYlB< z3O7|qAj;wDCK?n;uIdJnAx_GZ@561e#Uq=}CoeSmlPU({7cc^f1e=Y*XgHK%nn$Qj zGLhe+JkcLilOQ3SULKToaU+I<<6_f)X~2sqKtL30NF@ESnO!C0 zfL@XOP<))axiEi5=mA6FZ&cNmjxBo}U;fEaZKt|r-Gq@jeA4bi;*v-e5{aWSvmgfU z{(k&U>BV~1uVr{JbjP01XGZvaWDalCm>x_M*cPnmC6Lqk>E z{s?iJs`D`^< ze$kYMYAPLQx4ZGVfgYHumX$3|ud0c(%5#u)v<*M*T;6}}+4cR1<;l}0_GM44wMj3q zq8tdBlZRFXysoJM7h~XfW0OHa&0!noE``f?Yd0&}7W3;cArc&G*E8{sWaW6bqUYOv zte<&`AM-wCfe6>tp)C?ae9#Gd5KU2V;h3qK%Fp{1e?TM4&M+lFNF=aw6?g+m(r{6^+Sa12)lJJg@t zVbTW7$i@;2aTCzg51gYvk+E3+5^>o-woc6Fkuc1+usVlP)7R?Jw#c z9-f6?J{3RV(a-ovNmsA;#W4zD2|fqj)a?gpn>;4wnawIc4{%q*^Siik7#9IH2>Sf3 ziEtV7LwJydD|Y+!#hx#yX5Q1`}>gS1#+1QPRKn1sRx1Cv89f;^hH3LZO~?8+r_}AcP)1 z%TpXm-|kt-dCTt=IB=qWRx&Gzt|^yay;9TiBg@#muIZHf{AiA#^&QGr>V`wgYM$Vd zV%D(uQ4{BNL9tzoZ~n%KmfD|+`g$x2CzMxDn*~Bl_7fJ;IWK%{JutLzl~E|+a=0j* zVnIC@3#i@QyV<$=M$GG&lj2Um_{JEQt7pbz9znr3J1K-eZ=JJ$JkL!s=!dGPTFIKs z#wAE9u}hdrpVP-~s~@-r4=CEd2_SzP z=uI!{R*S~4p(3Xzseixz>BQRTViAq+XR-i0NMS#QdO7*X4AiWgxbHB{kv|H9zfHfa zNU+7E^YpF#+)n{pIJU5e$k;XdiO%Xp1S}6RH(4wwD;5iu1?Ea_C+Q~X11qJ|C)38~ zvjKVUn>o2?CJyQLDo2iG5qYy{mdQ%6YKYQKQ;O5(hJs75tfOHw$~aNL_n`p`FTawt z^`2CmYH#leb-UGrd+LwkU~l_%1LvQ^#f8MNzbOn{J3MA+jSbBPi{#E2Nrd;)c<}Y9VJpIgD&X{TZ>6#AGxs2m+dOT)Agb`nK#ME78 zUP{%^xaVz3G~q6I7g*OV!;kwPAmp^wxQ(V~9L#CUO0BG}Bxbcf{yFT8B=ft(A};g1 zUn&eX3#%vgC4cA+;n>RZ1&wv(fBb&A-}_ob!q%&kD8kO|G(WbVjyOose*-AT-u@l}(n0|5OHn&{rmSR&Z*WbQoD^kb{ zz{e+;D02v)&?c?vuO-E09vcmRGG$gXXl85}y4e8qoiR3u_CW@@qa>ksZJO@r41eJ!+6*+u5*tpp96wQ?4|FdqKSHZ<0+{=HbelZnFqv+-Z{tSC$o9H%CNtd8;>f?~IPbuumUKf44mcsKmRV3Zo}Nxb1>o zxm*S?tPaso>bd*~>kiy|t{pYITEeuAS2xt&(KHDdl4OdSg@QBBy>1DIsh@m{Bqh{$ zQYMFdu0UJbI-$bcN_wFwu5+yQP52>Yg081D=*x+^VGL|Lxx({bZ(2sd8_Tiqvj%r6 ztKp)HSs{l0{*#T4L2P)?5VEs!EWhr$rTI3B$~~~r82sxk=E@vQ4VY>pNDsB;pl9iO>32WrMFD2IqC4c z#L;`o=d-rM;bIeo<2Y9rC!f<IR@`H8)g3t5)mwp+pqVOeyRVSlue|u9t{d}lRe6GPGj%xovtU|3?2UZoH z9?r}d>UC~!?b?vb4`B9Lo`?P|#z6*vEi1*ct! z`(=f*`+VqUQGxPL=3-*FL^1KO+KP|8GMbuu`JxZwN?dMG3FPbzech}#diyTLR`E*+ z!as=Rbj}5sIU0>$q*^&t75sU+b>K6w>g+$1VLQL(@7(+*C($KlCE=8I&>>fqKSnid zac!3-vER7)NwZjl$*rkFgx1GsS8;sp`7Kt#y0wB|yHfnZMRK!(jEr2;f&oQ~-0?5I z&H02z0t0cEb*X#C43k$^io4LFx{Qp0&59(?%CyF*><@xg`Ml27U+;xQJ26_(U}snT z+0z5JhCW4~SdD2zGQ#NDW|E<*1DRCB{wk_s2JY8fKApk4NAClJtT*c7x2~)OGR%Rac%urlNIBsy>4AVA|d~I#-M$$zdJ2WBJ?1(`cmy;Fk3zpg^R+F1) zvPHD*htJ>K8^`HcUUJf4Uuo$Oo2TlsuZ=d43jLW#ln<+Q_n~Abn-)>|BTg6zyxVu> zwqa2u1O$z_Nut6cOW#v9m_7)iCkV3?>`%)(;lPF^1)|r+YhM#o}h|+E^BbN9af}JRayC@o&=s()J&JSTP3aDKV9QEQybsD(&NBziw=$N@`|^W zdevE&DErvmhy59)ot@+C@!f`tizM|T`wS;`S}yhl2PZ<#=gXB}MEo`6(xS#b`I?ZR z?nh!uos4P^ix}a)vrOyvik~x~CYcKiR2CiO^UrYF^>f6J=n%izC={8H$9`Oc_3g>W zwI^=^|9pIHJ5&AH^z?BN*+pdJ@cfvmKSP|N17VG>St9gNj(^h={-gt$X@f6i#PGcO zQiAo|p!ZK+34EV(aTZi31*qHn=ElY~QKBeBRn9a%9$u+@a0WM%(}o+=DG>_DdM z-do0zoTM0*j7AOXqU56Dl;@1*&Ky za^64M*)g-1J`)k>Oc}@W&3d=ShZ%)e)ru|RQO1ym`#+L1Sc-pf`QD~yVEo67N}641 z|DhQNM)M76Xqa9$Jsz1}X6915sxm7HR8{`d;@W1I#q}{t^$)QH&<>_ zrkbY(m-Nxnr|+^sY*B;y!bH(U*;TkypahiX5X}%Xmyv}l?efIb>htxJ38(WWr~Rpx zrUP#t*^^vn#Meh~r|e`!xTW&D=Q=!dg(QZJg#7KDGbMC*lfErB%Atw)VgCfjDZw``n&>ZrVaQ00 zM$hX|wkxKv4DY3dwa)+S#v_}R)AgJG8fp}*jyN8>SZFw)=GWx7SnaW=wfAOY5WB9l zo%yq^v1#laHka@@TVoxha>37iaX@~}Xr-bsLCX5PC)|L+`MY4hyk&e5oUepdg0@)GR1**l( z=?8(WBKQfEjwwW`q3_bb{*eErE>k6?*^G?u39M5X5^uv3*K$GzCtUBJicQ&E88{tK z&6%1W30>3ql_1+A@KCs>dVb6sCjx;TvFqHlWy2L=!&P%yWlL$dy)k)70bPYfrF`se z4i4uL-1o%80*hEbF-l!1P8`eKw6kZ_L>R!4$MU_BaXP@ZIlf zc~{au;s_rf9Bh2~KLmkq|K_rv?=MnyQ-IhaN^=KDJ@&Xr3({w^CY+~(^SCEkwbXL%pb@`K~P zJ;U~mP(rUTANoUsiq}b*qAwrvZcu$aKA%!;@3}f6Lzhx;M0+Z+5^aEKMrVG15U=6` z&pV0fVUn5gp9-8GDioIDTNUr~!(;PimmLfQtT`1Py$PEeA%g9;Yx-%-|f9JAWm1^nQHC>38;Vy-*(6;`y+CIdkweFPZUb z!V*pdxod$FHqqM<0rk<`>%f)5PqOAWsv~6V%m2~>lpHiW2Tf$RW|B&-=5ta|kT2n) z6&4A2*+G2{8Ma|nicS)$UCfkE5TmEJdx1`6!yiPp{)WcA=&rt=>zH&V)s`?A zhQu=ApjX#(Pgqp42MSoPVL5iO9iqPymW!(uE9Mu=$Ne+YZd=X`R((tJ$riaIP60jR zP%T#1zx;?rUVNT9;ztM#N7~gvYj&#(iwRlZsnhF2V8Rb+kO+353br z_{iOiV{jFrmF8G2A7@pm^gG5pSKr4r7I zmI13sJLDp*OXRz=x7#8Z5kFw98na97+_^Fr)A)Kc=MTldJL*Iq9XZ>6y^czs5DO4EU8dIUGi#d zO!x-|k#&DWjtU7U?9z|;*%eJikJ8dW-2Lp3kukJSPoY92jbb_PnV2{W#IVjl0EhOrJW9{-*n(nL4LF3e)* zX(XV_=4M?m zP7NY;&7(W?n{g}FJ=r5ap&eFyF_~rW_Pa;9_!hn`)!&d&GrKA6-wh9W9!2wh4zFMWEMId+P}|Awo`#bu6tiYxq_HbN)J;1v$9PJajF-aeM2i;#7o6Ze(Pyz! zd-g~8^Oa=@UmW<&OEms1aFKr3k-y9UW_1^xN!CySw_zd76gS!ib^N!IWsZbykUF;9 z>|6RnCnqQDFpCLGm0eC)$Qs%A#9%S2@@4`TOI#VLXie-V@_64<6XCVO3>zkC8J2mM z--y2t*(|-&-_M!D30Qa4E`qg4FD>%(^IHbgLo{APPjZGN8?2q)2y77rKqaUE1N_d} zD-4cRreRcHr~Etjr1*;W6=Yu2hQ%|ZDUH0sUurcm3+L==`F>6uqaYwG96tPhNm;|Y zbC|MnZ&{O>8nKFlXiR}!Exr=<5EFOBH_Y9-qVG)$FfsdvfTF$_APzXKWP8^^-r~m_ z@1jv|3ENy!9DnfiN9gnH%kQO;^V3%W?>AQT&2#168+&q`U zulubL<-jZ?%=^p4w{17|n8^!G)8{>M-vul5a9y9v=zB(M_Cq&R#rM*>-R70r@tj|& zR}#eJuL;QbeWpsRe&8|{3;JiEX-gT|J|czepZ!ZkL$8&sm4lJOy)L3*OHkj@-8w2+q=Uo6T+IP_c;!n!(Wkz_KnE>>tHnYm<#lNf zwjBSu#U!-#Bf~82z&$+|ISf{dCnyVo&j}FYBn`Hhk;Tca2fUpfkKfquKG1#_@HDuy z;dF)z8x;uzuHsSVgoU^L3?vx#dpgMVgyw88|2` z-l{r{pSER_P%VD5Q<&#$w@l*<%TD&~8xvludd;1wpLe|X6f{WO9ax*BzeJ{#Ct<%j z>jaUNdQ6933d?%N?dnAE6VR`C!0@{%iW|?@W-iB66?_J`KMU{K=ffOfJCq{!pMSzi{tiH-OiJ0(H^O3Q2Gvvexd z^0;ptW0zfI$IW8ax3@^o13*(fbMwKV3VidwWkO7T(c9yL?5Om?MjA6LBTR&g!#su${*{xjvl+=aCVtqO z*v6Siw_45%3;r#l%pR8zIUX1*Uul5!E3&Gb%&^R`zq5YLu@-NMcRMiU70zp|)?}R! zv%BHa7E{YX%{=ZT0;r7V^*jekp&(V!F_(KQtMM38gQRSPZL^g zHp__qY>+B{Nn;nsvkHYhema2DmrG;U7KKU8 zt&-l_)D-Uhbi$Q+bJ@DbSO9bjC^&veM@)cr&Ql7C3Zq$xtqd|bhgn6Xj{0qhM-K78 z4GSQ16bc`+seB0gE5Dk)|4g?APk2rSP>+UI!Nff&Zo7c*bdA22D%pXrK`p_|bBig_ z7gZA{xG(_fPcm)I{(%!gnI_#+)%3QpEEX=4s44y(Z@ZBr6XZ#iKB&odX$W)Z9LE0# zg+Y40(=-tZg`l-YI2=B>p{5={K@cDag1RmbT5AMB0HxG@{w4w<5r_TwlZbpRgIsA6 z$`OE8a2F<#zw|nyr@jM5tYP2c;QRi4y}mJ|KwG?eZym>!=twyGy8Ng68lVen=#W_q zHAP?=63tN?A355Drig_=YczzdUC(!CYB2xvZVkk1t0Tk zCG-eXiej1+js;(0y z^ne{ZTbdAh>N%*d|0VM7BD7uZgHW8l0%3&^J^2h|h{u5a(3c9q?2Te>c6H0hZ?+-v z!QKXF=-cYWl6M1KUn(Hy1sG@wF&-)83H0A zBHHf3xZ~{MRw@>-I6sF-EC#n!VnFVK>$ylT+<-I;baZz!Aa}WMucp&*T^EUlhVt0g zOp@%0QVPX<9+{ODI8F$fb>^N)DUoVwD*xdrAYd$s_QEtxyzs&cc>3w5v9hv)hK2?> zj#IbsXliOgB9W-;-f4zmpuN2v_Jh(3R|tr%4uqe04(hePLQ$?ko8=xQ#mS41P8gA6 zAAqp#?e`6ZLg?%3tLyzi07x|?VVTB`b3Y-lBRR3lCPXS?;rad)o*PJEuqg~F1U@v- zh>o~}zaCvdDq_`~aDBIBRZG&W6nxCCl`zyCLDZJGFqea`G^7C9;ttYz50fj!+fI@Q zA<)^`3EQ@L4oq8*p$q{Lpp_5T^`I0WqyfveU`PQ#!SggM%YfDjn|o#oDR()9Xav3o z&j)PBh7{QOej%%mbE);#$*6?bw=pU2c^q$d9S_U*d5E{wJ>G$^5U~Y}Z2> z)PaXm$%o;DU>Q7ly~}7qE4}GLML0plGWlR7ill@bJc3C68L0QZ4p$VQ4FT=DD2}}c8Hyo1cp4D$p6?U&+~ZRSfp9ns z$8qjzp9ot9K0MS2DS()5R<4UpDR8_siq1s&`pq%#&GzeVaPX5K=t9gfaB)6|i}Sg1 z*_Hz5y5bmYic}rit9J2|NG6jo41-A$+JZ?Es%y$CU}xfZZ~LCr%$h#D37bNGlYU zXEAc=3KrH%P(~EpM^9p?xA8tzGz$5pDO`E~8q#?e+D>5L_-XWa#P3-HN-1*#<0Drw zy_iFw1>zmUICZ=q(RvyxUuzVXr*P%kIF^=k2q2N{J&m)++ILLS_VBKwk9>Lx*RPFY zCGSF55p)cl#bA5v{_l0U8yFoQ#f^mwJPp{*NAcM49)zrid`N|$i0OB}f-ipUA`-`6 z#7AB{gT@etiMX?Js}N{WPWT5UEKXd-<&jwweFZ1giQ}hEqb16V0(zLsc!1TZE4X}Z z5^E&|X|>?QGmoMzY%x(nM0GEl&0=zL5`BGrh{a-c8;9B1Srm&!3=9m^?T}-h=V5GY z43S6#U0q!d&KEOH7>^u7B$I&}e*<3F2LuolvnY&w8!{4u)7}rmS?XexhdhL3t83ed~{W@Z?7&c(8(%&^di-nnwg2_!@VQG_oVXC+cmLJjK{7LfFMj5q;I}^a8j7hQ{Mf(v1N_2| zoyPX+*vl>B%4?s;um1C2$6ND7bbkDw;E#UoXE59reZXU?P+FVC*MIw;;M0HnEoAIr z{M0Z08vemgK8J+3Ki{XgGKN3@^#6k2dg&ta%}?Ox|F7S`&-~3(_o(92aIq77`}q6ce;o2Y*JECO30GeGWBks)`7OMk_R;;l z{~W*di~k+EQ{jg?uIz;`*-kv{`#-uBgZ&*0zJf~JaDlx@)~~oAN}w6lQ$QT=>9l< z`76JPztPjcBnc6*j#LVTLZR-5PW3zw#bU9(TRyZ#sZ@F}dliKoSm(|nnps75=_34C z`3UZ>Eg^sDHAp86tFZ(3q6Vp) z4;Hiq(`8hb+{_fd^6!5UKmT+86rcOX3ZD9je~5qa>0ib_`1!w&PyUbp6Mps|{crf> z4?P9UjbLIn10@|a4?l{tr~Bae9z4&5zss@Jup)@ZBQVV{lF2Ac(?MhZ8GPXEFk;e& z=X&r}^${AK)?qyJogcyRmW_%#s5zG7gT+Lu?<}5vd>A1^!F4_O_jNpJ6HP-O#N#Ks zV8Msyx(L+0jHE!k>k)j%N1sDa*nsbO57D-|mq!s3$$`i5!iS$hd(WvO^xcryDjZgpIzlV3Hi#YuwKZT$E$sfR@-O)HF57+acg#j&0Z0`-Mwcc`qD};a$K;XIX0#TX!r2Jw9-*e#wfN9&X3|Tph zD7Ah50w8cpmDifa-84oD80IeHI%*BybKwU8v@l@X_WIl}*fF=l#`^92zMDT=m)GA{ zfHYuP*3Hjb-xj86V9PC~syxpDObfPUR=L|$!7G)MU|VCid0TmKJs*%3v|+$}81BH( zs+`F213+$zJpirLw)qOmWy|#x4BM&FeN{vG9$YVgFaRM;7!O%HcC~p%pvv1;7&ffR zK3#qP-x}TkzULuO3c|2pSytt>TVq>}uaM=*2{)EsF7M)Ix%NHyez5a?lBf}QC3wDuQKPPdR`6UGz7nu(8|ERH8%D1mx3x6k zI3|>@AWZ|JYWdps*!p}I)#ky_zFVm~Z2RW1W#?-J^hVujIqjllEZDFNHDAjSG|e%rammbscZ2hXA4 zc^we*NEKYHAY6WD-C4L;n?i>?eK-;c)HVlZ8S7U-;9{;s5=A{ypaB=h52Q zivRW}ehT0J13$RyB*tr*4F2#x{67BRKm0y2nGAY*d+~Qa`P2BwcYm~QhwPz;gHj4_ zyzvHJdF2(%&CQ{;wG|)x*vD}6=+U|?yN=`3eS&ji9*PHM;`VCaF_nb<%yWo*t%zJ< z7AjmmFZC8Dk-zv$SfG7`V5Wf+EmSS(ieLo3;~jYuT&P|n360EDbf^UrCZ;Hf?3 z&>r-Cg6U;P@s`POxamScGOL(k%PUo%W^6_ZylV{*}jX)0tgIk-Us!X|1C|KP37;nKHX z$Lnuif*0>XcN>uR^_Ip5FTafO%kN`qVGV(_(A0Ab11%oj`tp}>X~D+X@A(9t985t3 z15q8 zd-8GgwM0>F8`LN*Pvhd7ui{(pjzXsfG2Awf;{VUyc}K@pU3>VuWxA%T%PO`d_uefV zOfjZ7^bR2;0rE)ll1B)S5E5z#Ngg2yq=&pmAR!O}p%bbxHa1{mT)>NGS$pM=@137Mk8z#ioW3p1L_KI%Y1_h4$X%H8gqxd)WBKYBucX z!d^BKPdb1yqyjiJlA%_b8XAZs6%4t6;?g40EgRUlbuVIJEmLOAV{BCk7FigawJ}JA zTiDyQi{|ELI@;P%9U~Y&b3PMmt8tiAx*9jp*cw974M4_IUdyOaBajju?Af)GP})TC z$Z^zE_>s|wwe4W@I~!^3PJpe5vD4;IS5u0$&p@4xx3hE8n{0Te0mC_hQeT#|`UwzU z5I&QVHHMlY*xW!va{yhEP!nO|=_)Knh=#3OX$vLMu~J+!0&8*)Z>@Tp4#mUdg~zdA z+DPoCBj@5Hh$50KBgt~F6Ydj6HrT+65B!=te{nxsy9NB)-sH7q8LCE)V{Fy_PpZT_ z8(8tqYTjvzQ8;ckM=hL5h0mT#G1fCN_HJFz#&?^DqzzmZW0*c?29-V=pM|eP6og({ zO41=;ZjE#|o8MT)+Rd%Vo^mQi_z*P*laHQ3K~EPATes2~&7g~RMvR-tNKclAEj#Fl zq%g2hK6w^nN~|Qi_pp8YZlWm_+2*9Is*=R^RlK=r4`R_c794jprQ*%lPq?I6r33Awdw3BpF%O5f4+Nb3=*K+OUJBmLORTY(12{nn>C+A|m`RB*q znKcbV*Fn{h5C_n+aaRMk-Fg%6zOxBU(*Wr1>Y}Wog40evbI^CLdE<3{dFLH$-?kN9 z*V(yaI~^SzjIJBQj2SZysGD_NXW7dya_1dCqrJTi!!X#nV>^j>f|{C9j2k~bzgT%3 z(Q@?4E3a_pop;jH(}T@sqokyS+S=M-=_DyFEk)P$VeQ;BnM_nwRUO%Tlpr9KmtdK- z0PT&JNhuKwlYpUXWIA>s+6u6bpNnX=9`Yv~4hMC0b;G`KyWj7}<#G*8Cy4-7S=v`_ z3xkxZAGD7|k|b(sYI^S&`FJ_y<3_;m>T1C`|a$ z-Q53^FH!9q|y5oV!rwi97$qV^c5Wy6eBjw8Av) z?>@5_Nx7$R<Wj=hHrkd>L&42!eU*5lt zamSxboju6Pd;i28;Zn}L=BM27wZ(X~9(KNTFF(BV@9YZOnRDR}x%=m5(Y*3LuD$I+ zn!*zEzV=godCO(gIu+VBKFi%V|A;5vky-l9uQMz2Hov;{|Jc+iGGfAPjydg8zV+W{ z?4w{R$ol90PPeIxwJTP#Gbl0j{Oh>$M^`eb%nsQIn_sw(pWpEy>Wp(Zr8dEXKmHAm z)?djD*Imq*B0GAdk;m@3mAn4(5?a-K7SHcw>nrcl6iFSirEBz5E6e}ym;CwZ?TkM5 z491$<`1f!A%1&bx7ysa=Ty@dWc*QK;@4n2hZu}X`x5e>P%;LneClcSVmep&wkhF~C zobUgFo4&aOpKOrmc!!66b|ZH^_!forr?8-|nXVo0&>R~|QCd2gSPLufy`KACdY2w^ zIkV^0Q|S|F+x{jSwgoAhdm-Qc{&g%GU&0_Nf^_>@{`rTyd9}&JypxVY?s$X0J+ca+ zW(i;a;mw>p%}4vYFYxDI{)RP8QDnzx&b1 z-7+3e>}J+U7g19w=k}9S=MJ8G;ExC$Yk6(uCZg7nTzJDTxc;k)@mmFqOo;VQ{h2#| z^(gZE3psH_7x&-tJDxc1Tm0~va~bWked-Q0QIZ%rW-L~l4R2xLp(sN(J$4sA{P|P3 z7JQKt$M4|DyZ_BD(-M9*Zz|?^CvV($Gq*pumKY=X>Mwr7e=T;i>Vez%>Hob(k5I25!{){_rI1{;Z2hZMpJ@>w}ouuku)?QQBWzn+>gBXEAqyS*R?lvh<@b2#w&{fDQ{ceaP-O;2*`fB%L}T}g_j z9LK^*RVa}r-d*!1ol-4lUG*cr_qD~8I7Grbmh;D--^J^}YEC}pFGIkjEOmy-^v%vuXE3RPqDYh&KK0t{NkMV_lDCm37THH zkL!QEjKaC6bMd7YP*G7qf#nmqt|OaFR8&<{UR|BfF!(rzT~%doV}qJpIYu13#T2B_NuD#?z@}Vv3+|klH4#1_U_ru`gLzH zdGeG4Mv-co#){?3>F(~zWpV4ePQ%XayuEI1-bs?jk$u}W46@lQ@pzncI*q1j!@BN% zJRV0;6#Rbwur-G2x=u72MV4he9?y|Gx=g8P001BWNkl#%=-w@}7rv%IHFgMp?Q zSp4Jo$`5bi!eho@%5<^uwdYv3EsX50VCqo|sBs!(vme%{LZ@}rL;UU!5AjaS!{RSp z&G&D)k^i`;9`}HaTg2`g$*kE^@!NYd19om)N4sSzr<`&OQ^rrAwz`niu9e*Li@SMo zyUzRzzRI~@yoh7QSXlq~J^c0Qjik(e#xI;hjnjmxWQoUhCY*2~-?{GV%q)@UXg^dcLYI#CRjuB|Khj2oL1?GZ4w)M=NW$goBLR` zJxcL}`JDdcuW(eYgLv~go_PEPnz~X57AIq-&Su;QCt5a5T2UyRbQa&c;Tz1a@sbHN zvF7Dv?270ZN|Mdb-ox+je~u3OB))vr_qhJ%>p1WDI)+xOa1*|o+02|&gGta(l1V6? z#W~;l5#PP~>rC>c*zxM$xbNYY2n;308_wFlMYq(GK%SJYBjIgtfte}!YFRAWkP=czycl1ElFQdxgK zr=NK-OBalyeeI(>^zd`EB~@BiKfqt^dxV{-LXN%gD*p4k*K_e{Gb#8440C3)mK2VtMr+&$6itN+vDlOW(MVI*$n@ zny%bhFk#v>$|@@l%f(P~lrZV21=PAF z6eUYMX=3(ySMrlz{e-WaIt`e)xZR;L#OyC@!X)bTMB#a}h0;aMmCdV_o^4z$yk=2v4qK!rZKL@gZaJWFhDYi=yDN9 zU-Dgk_@f_i-s0&L7JM?g%rFeHnG7WrOp zP>7C>j$z%GMO9TgIywl4!$9A#b^(1-*>y?rYV-%gAm z$ygjN3d9VtNQ`7p3PI|10%bx0!r5NuM{gOG_s~s}kcEL3kW{>j&FeQ4SHb2h#A`NT zvfJ@^T!?Ctj>ZmzLJ#{V!w3b%OrO7i8I#5_rltr<05uXK*po!fMA^CKIbMCM6~j}4 zUrLiuG^`E>Mszo;m#;^O?BUJjud=hP2dirY<0jW&kqfAvGM;jWl_U1T%ST7h$t|sUAOt64%h%QhG0rfVGgZQ)e=&vYg7X3z$~t2Q|u$mCI;|BnHV8 z5|AxU9L@qnHBKxXC8_Ahf(TlQSR_U!YalyHnSRWfOdsV$O9y!S`DfV`S5U)SX~>Lb z{`8SpB?xa`!SXflki>+`>meIUV6qn=s4*HhtzuV8E9;gmqoKP8tG}F))io4Y9TeBj zpwjm_GtA}q@jXAA&8qr{oWIBVM39)7M^Q_v`0p1e)Vh;&b$7;8u zMH*SXeB+R;?QQ63(&-G@Xe-bE@kVa`p9cx}7jV&q$5LYHl^@gl3`qilEb~6;5~N=K z6a*};LZ&V_fhiLvQeEam5)G1xI6a9T40?NzNOCXz&5j!8p1g>O6Gl@}?7@UiHW?%w z(Gf%w=RpQm;@vnP$A zWW;Eu&6`E3!yw+ik(ZWlC#wsXYy~(SHpE0X-H{ZE29hMAD;Z+p5GlRyok&Px?=lncIc0kdmdQe#Vxno!kjsC@|kXV3>{d^NONXj zubhG@vhQH8C1WJFzK+t>42GVk({Mc#2w>f>^sJ&8XnNk!k;h@O59LXa$8ckLkJp1q zmH?G(B0)TzK?e@z*eHl1CbJoVebH;Wt_>&w@R6Xj(a_LAI%8n6Sh3k1ea0=356T7> zWIH9JCNR3piAQ$gDFxJKmNhmKjAlqE+j#b#U()Cm3A9U$96y0_pNx_=-g5~wA&Fvd zgIclIz@TaB;q|5qGX9ara_t}Q=j6r+qTa@{4?fAu8`^pULv$UrKSV@0klEz`t7x+0 z^}CS?_cpjT3=CcG%QO+NIt#G8*GVNxm`uIKuH0o>RZ(;uE!9KE&TVvNbo7?DdE^gw zvD#tK*==U@)G1WjHH=7rJ$qY;s{%fglLD92e{(;gz0K}PXY=Dfjbd9%6hUw0se9jK z_0Axw-WyH4Z>{&yOAt)RCZVtKt^gS5nu?(rD9I*vG=$M~uoXD5nEK>E6dB5WaRpIA zmL)_%KsGz@`8)_Tlj&-tIg&xIupE%L$DVpEw^c8pGohfhy~w}TJm$u$wlZR!?MPp%Vd>j@`THj>O3H7`*-OKk=s>F0z3D zrK84SD)k`t?4qfui)_}wYO!In_V$Pogb&sipVQ-UUzD$aAPT)QXlQ8ap*8LsUJwu^ z8CjNkb-&QB4pj`@K<|mu*07!Kv<7?D@{iyCl2@!6?NK{*lP6PQQHErMjbIr?{o;8% z@%C;SS~l|7UDwmNSE^ouT@~aX5yMeM>g=HEI{EINgOf z?VyE|v@|r5U;)QmegnUmc@~{<4R!4U-22=sw1(1P*wA$i{Qx?;Wc+l-7Q1jc{kZy) ziS*>YXAC9Cn-Aa3hQB^eV#1d=>5D45!DrG#Cy&D{zk)z@O$|5P{3Gu9!*6M7YNEKL zg!3=Bh(*U9_kn*qYxZ2Ox%N8l|I59!wY5?)Vgy%w?dwdKIBD=5MwF;ucof(D;0FHj z-~)t0A;yiLz?Z*zIW;3k<`*Q7BN~zBT=MvffG?%lf?HEI-IuXos%n(EJ{8SXk% zlBA*KD0A2`&7X(kg&2uM6RJZ10n~Vio-MB}+MjKgM;-(CB2R)mhS@QR>SX>ULSuJ?v|fedW1bri#V%)P z88UsPWO_I5k9^w0;_?hMnCL3mOa@gGu((U9Kj&L~dA0`(0|O9c3-$s5CHP6qCElM_c5BK6TulC78nZ;{Pp`wTY%L*+~%m~7Zg zSsGXTlNYvV9CO~O%ozC?YnoyQO?m?`lr*V~esKIc)GiXDh^g-~W#}5p2b6D;t#)Lk zi#1RDi$-A_r!1b$h-X*Ol1TT4KKCEyoTFH9>~TE#(p$U}*vaar?!~ArW6V*NxNI_z zA(PFZsyc$z%b5Cex%|9oNc1{d1kr@W*u<)=NpFT{Z2?7Pqku165 z+ZkoQ5PBnk2b_NFcsw4gW`k6slcvUQtR62O zpAV1Qg+(;@RI1Pz@Bbm*zKI8J|2{vy>)$vQp34Pi)Kh4c4zS_%W#;Vn4#O~bZ(@sR z^)U6+E4lWXFEgV!$;!tc=3g(Y$36E;+%z8E!Xoue_2iWu@`dANz-v#5B;Jf zT8k+3m=Oeno_LH*R_UGp=ctyarfH-z89+dGl~Uw39XLzIP-%JXUVeT1jXd^tocU*5 z#=?msFdvBb1T&R$mat$-4OTtI)>j^7UBF6Zc?D(>K*sIyVzrpik}=x%?j~*X;P&|N zcwAU5B8VP*UK_Fq*>sXrDw*TKJo3n~+I~C3e)ji|F5?y}E}su82HEgVcD8C1xIB3G z(}zQ`cCVpFY3{bM^s3vq{kCg3X=XJhHBQsI=UK6?9lh_u^j_&TK2;qrQcyOAMW>v= z2!}y**K!_x^i2vT&85^bL@#iLhLTOAsvt@xyd`BgNU-s#d${d~Kjzsj5+`198S}>! z4=9m&;G>WDft+vkjNy#$e2)tjPr#gR=heUdnunL~Afx4VV0QAoov@`k~`i)PbEI`yCg}%VzCTsT`mX$vMgh_ z+lQ=kKoA71rv05H#z8wtpx0?;Hkq!GjXLa#(w)5H}_wwR~R^qWREnDB@#pmB3r~>FYo2p^xebK{uuB@l7>mSP?Wb_gzM-M~G z(!Kcw{`K@4f+-zA&k*ixp|vaC_d$hpGKr#r$?m{zG9ef`nPh@^JVx`5J%rNP-U#P@ zhl!yQ+`W%Pl5OJ-7NH|?YOzRB)ezy2~PMlUS{n~UnH zbEx&oD2Y8h|ChhHId4s=x%BulvU6T z9aUA1uzHvpyH`BPlgl>}$r=cHmT+4WoslfM0lJnYnMk1;`|3hHPj3}i5?W)Al(zBtv!Gtnkbt# zk1?JC!~wk2M=!0Lwf}yCHCx+I0X>_bb8jp0tb&1os-#J#m7F}u-ia(eaS6Die7 z)#$O57TNlYvJz$ErchT=gp>`lW#ywh|LQx06EOnKTUfK~1sW4&Or1Fqw?if!Xk%An zBk^pCSoagzCLlVzjGQ)$k#31htdVE#eSkGh-Nd6|8rMC~3oEvNCXL%a zzKlm$|M-KfYBe$M+;4Ns?cZTRZ6PK}>~+v1V0F2$h$4oTr6-;s9u2U!p_#bW%aFM( zyqALe?Jat4UpMI2r~7Tz`=1T^bnCu13=9Mi%wDD}T*ADGr5I`!RUXNl86`vRz@AOS zh$j_vH9@eenQ+{|>K@6=`4h3kck{|qPqA)i5KR}*GEtgayGUhJFm#kmid0{8iqTKA z!Bs_$)}rK~Z2WvKxbkZ(9P1*{`Zj;R`;RQ&86CzMc)ev*^R>G3$Yb~+nM~5u)I>6w z9M*K`?(U|grDaI=Q#)KxRh5>O7P`8+hTy?gL>O6xb=rJv5i>$xhQFbyq?@;qYTAUZ zWInP?G&VMpPN#?OslYG@hr={BHV)y&?eEPbF-R*1C*QiR)6&vHcX#)&)(v^QANg`y zc?>%Ul9ihJ^SJibES6v1!^(G7^V^$mA%4Xr95s0aRzV}$zJoQdy-AyCJm)VSi^pn^ z2?c0x50KFfG&N12Z4ZHNC#{`P6cw~|g3k6fqKRTUn|BjPsDMGDb1y9&AqvGPt*xCT z6$4Gm5NO#$ATfrCr(D3&7vJWAm$&oM!}n65ZzZ*T6}#eD1W`uoZe(*~J5`<{+FRR5 zq!rLqdID{<1>zL=o%HSzCPpq?%2{7nLCXU#)3ovdZv6TlW=$M{%T><83ofTv%+Rr? zg!|9qY8FaDA5g*P*2d?|9lD3*Tp z%a|R-)Gs}Y6P9n_pReuZ<@u*d>|=C%Co zj;nZQ#wY}<8HpsFTUYSfnuS!II1QIo2Ej`8!qb^w`z&c?Dzj%!!L?u0VcW<>oOb%j z?Cf}$ox7HC&#m8N{j_noT^`0QypZX|9<(JFaNLSbJn=>&%O1NIe`*7X?JH?cs=fLm zu#0VbyD0NkV;Q1NVhmj+*4au&Ac~4kHWi|6PY1DLH+z~pNhul_X@YI-ghD~OcefGK zbTG1nT6PnN&B0@}F)V|EL>X84L?(_crs16co1gzX56jz_F#BZYl(fqTQ7zRqTo84_eW{j?7!h$bw>e7{TJhq0-Pyd>0yVo;)OfeSs7*4%x9+ET! zV{8POp61p3^S2t(SI=Uk-HFRt$mlVPm_MToQ4q1$O=oJApVoJ}So6f+_)Y7}bidnx zq8lKj=-R!9P;3&i-bF`u6h#LulcckC4}mU)*6toO9kgVW_Vx~XluC?n2W?$3Gz@e# zNNZyUK|1LSL{Tvi(lI()+el6wjomB)BEE66Id0KxRy4iK%#%*0)-}|Q>jvSiuk!f6 zCXyI0^1{2G%V+?tG-SBqyj{f1ySo^&$?y2`ub56 z&MBd*xtUN(2SgH`d+7*7aoK&Cgx>GOgYC3-#d&<0v7=}SG7$lWS<({!vEcpHYPc*~BFpS>J=yV!Q(=ZHU*hWh$ib5un8P+^& z7zWvF7L&;|G$$ZSh?B=-i$u|OzDd@jV@Lvqk|DKg1ESS|wQ2%_{K50nb)9rNJ!G8& zhYPBz=IAgK^@1n}0~)sWoW$<3rwuHYx*=CJ(V&#`*l z7T$dH9@anarod{(;w)l({b~H?;(1hftcZGsU}p%qd=_V3HWpp7Qc5ekcR8?(s^_dr zC!vd0e8lN!Z=XCI=(s80uoe)JVI`hlS)+7pwb(;4Esqz$1(0AdL zJQFzoyEkDi8^f|Uw-L#R2##{<&$^fsXI3Dq0XkZB#-DO7WoZ@BSxPe0>&U2E@ zDzk~v)lbl(7c+b63`P~15lz!L|6AW7>3o#UZ5b4!h{YFOPF49tc1Mz!eRkrZAWD%1 zNe~dVZdN^bH*fr-e{&ZQWeX(}7xVR-ZsN4s7>t<9S(l7O6)Y4=Nm}>pCRsFx^S?X} z9TTNWoM<9TY58=%{QV!{sd|{@Z#NKA1XNo!r(AFW3&)lq3Ig6qOZnCf8vegO&Za#b zta+=I#V4H3SxPb8CJ&XPMk8nAz0ISKh}3qGnqw~Fo8MGEwC|^TZ#fZLB~_zFla55m8h+%V(!0e`&bX0Y+SffvV^bS7$6UfW zCtt)V3&&%VR05ry=>Ex^a9$;vV8(9>(-n6z`|K;Z$S9{NW@glkB^-Cuc!IM=^Ym*w zkjiEL~%j2cM{`&i*3vE3F_1 zxNA?~Kd<{11&^$xGb>^#n#_gYo=(lE*9mC?-hw#6XaYGFrl7W-b1t8XA=~lk?KC&4 zxF#*;{F3rSO*CUtRE#%VeHw3KC;NGMEqM+g3*BE%zN|2K#?yu`1rxt_nixP^?Z zoMSG%k=t*&kdbcd5!s_f6h#V4N_xwH*zCwd)tHyZ=N>Cotl+-;?jsh9z3<;nIN=1Y zxZ(7|$AI@o>eqe{PF@b=qpvwHPv+S=MEEiL85 z6HlbBt}frCIFF$VlZ-TfHjY#ZqjdvWj{%~9o=KA2wg%B+$5LJk`O-cxJp~lat3rFKF7@l=F4Z~pND=*R3+Kk)lW%f}= zQCL#?o`-ZYJrRQ4A$l?zlC=O|Q86CJr+GOuP?Avsff$n0hqu5)D&B*sz=O?v@L`X- zmL}TWO)MkeE-c1nHy;+G)~2;<*|K3h7ORcv^X5@gSNG9ps41i~Dze3bNfH5_Y$}C< z37d7l@fkX)Sb$(m#py4?WjDS5GOs0Rc=bMRxaJPtP1x{y97q^wN{Uc>GaV6`Sr^~T z@Ba8LMjfOpSXYvSyMrWD2~XhxzHjJcdm;ovJ(yg6Ja$OL6>M%7)`Rc}G&eS~>gAV^ zB#H4;r!isr^iQlls;((yGAbsk6nWKwAavkkLJ!cbB~f&mPBAs(y3rYC>HM(4SF@x;0V zMk*M(N;29_B%xq&72ZLk5f`s&YYu<#&iTkB^p|mZGXFpbH4q*w8XQ3 zh-c)<{Lj<(bM81FM`ALGVHl{2g0AZbnt@4_^L4`aGXSmI+Q~DoH@oiS_rK?y6QU?`$t9O?;)y4IU{t@RX*~JllRWm=W88JuU3k48 zy*YI@o8@Oe`x&?2emm)O8jHok?|=V$zWBv24&6+GkAd#)ZZeq+<>lqWwikt}s&sU8 zV6j*zDk>V|R_B??K6~WU^u#KwcTEqoD5umZkNr`}z4D55Vlp_}Uq@?CkP00Hkm%CN_toZ+|Vt;TwFfau-$bX}boMjyJjc`?s??Qo)zM|6ka5WSKl| zG+w*>iO9B+&63S#QB@UVP;sBtm{BovEo0fal=kw)_Yo8)K9uJ0L z4C`jyk|a@7RP_1CByu}3)z{;AsfR?O1+72=!$6G%NN!()Xtp7H4(tfBTCG%6R1E79 zOM)QabULxwY(q8>;lP|_Ns>n%c?>z|no2sIB;6AymPn#e zfJG9}bcIADjd$F97M!)1^1O!2V^A0<=_p-+D2irakRcup5KU&W+RcYAQ(IH$Y2V3~ z&FhJ$pQoBe3a6b-t=E!Yf;{rbW1p%BwX$=|Cf4oULE-%+Oqn{3Q)|jG50!hHo+Z}R zM!2W9hgc>aCJ>5{F0md|mSrA?2ZAJ1zTj*wJ!>-$E#FSYQp#~(y@Ul71&Dd_eW}Dyf`T1OPYb(SLik3PU_M+%J=sF}?6bM}q6lxbWTAKlN7s+RCgl%e zdE}ADKA1<=bKzxQMP9ZU+2&$;{d}qlZ9``KdMe1C9RUhw9?!`WRZN8vdm9?4ttq6! zX(F%jKK-+jqkI-$``JAlzqNs+y__+%Rk&^Wegt{s@p0gGzdz3~lgY4Z)hg!9nS;~m z1Rx%d^V(~#v3BiRR;*aT)TvWB@4WLk`|Pu+tgOrv@-%&AR9sEdZ4!b62<{eSaCdii zcXtTx?(Xg`!QI^@c(4G$9fA(-+>_`1?k_^tnqlU2cXid?yJ{V=^u#4~S%2qx-Ylghpv_|)0Kb=*au=?I`mE`QO!PUaux0(|?9AHP z*)co;1#C1JudlDGsHyz}>L8dUna2CMC4~x*D@u1ZJM0PIcnR3y;pY8O1MXRrg*~Ls_aqG&8#6y z%CDXCtR~;j4uR##+i5YY-Fld)HJ)Di^0^bPbe*Rd4=!+Vw0M`kz;<@{*%x(+!`g8} z@M^^YUCj*}^)O{$^55$SUC;iAIyo_KK8&zChO$*Ij$@l=_ph}(t#o-X%kw>gS~om_ z;`ZX8jv2ESEb1FG zC$~n;TEOO{hzu;G08Cmw?!N<102sN0hJj(klmYYkQ~O+sYbSo|sw@uyd4j~vcSOaZ zi+f`%sBL?Wduu_#kG8E@0)zr&Nyog3_bHA_Xdwm+%a!lXQcoOs2!Jifxn|UG;L-_c ziquyuW4|vAfEQ9BMZ0-aWF!ER-rU^mZJgXcCx9XVJ6ave`8Q7BGpsc>(Cl760>G;~ zj~rkPR!Fhq+F8GP1~Eb^i9$+B3V1&aY}=n}11{28Q(}NCZAAq_I$(kV!id^G+Q0P# z-c5lb+3UA$>n_+mT!a{uG`8jy6&2;Qg8#~LRB8ht7&&qf222^3kKPoKLL7M0yeDAy zIt>~T;)FH4Ni*hnN(TYw|X<7phHUz4w zef}7P6LWAtD2W2R=mIxwVt-)Yadu@TLwnS>XUc@>qNsZDJ1G|YL2k2FwR`u*$1VyO zq3L>5cqk_;+r|}&Y@S$aFX*}H)?eeF;U)WisVH@B5>*S2t3wnf?+#nANKvH}0&^cq z=N}XH%KZ?Ahlc_B>7&&-*d2jF9&o-EA!hV%;mfmA@c835C8}85*5N2afkm_mF5n+d7BSn2R%j zT8xR?2-zFldmkir42%l&-Ct%x52a4ga!hbMwefWOA0!1Gm)uz;v0NNZhOy$`;5WoY5yF%Qc_Z)33$2g zI~Tw29~H$-r?hYa30V7XEVuhnL0DOB?P0dlG`7N1dGF1{^$dUBd{%&fzhaTtK768D zC3@VbFj*4dzPJZp4WvblD>5uNq&ad$6@Nbn=Ne)G3xE_MLP0?Rka|Y-8q@Y=CtL_& zL4Y2d0_`s=$`}){r*)SiM2Ij_~vA?$ewpgq6F>D6M2DmwbnM-nH33>#OVu(|-6 zT&^?;z~?yd#Fkt>3-)c>6T%4DA_&l(07S+CJUDT|>XI!(hSRu_5C&-MoIfruEhT4% z0uJN+u?W&<6;)C^g!?<7s}$Jx5O`E(jTwc7g==k3Ug_ym2WySbaTEKF338>-zb0&3 zjhQkW_q(4gPaNAFZ!Y|5XAXszT!1hUYPfe}#&7c+GYow3()kQ0hMV~bqk6)LD}T=A zGgogu{ps0xyCGMu9N_L_krkeA(z)px-6aC>!vo;>*>m897jhT?+jhvdHAw>G2zBws zI%C!d;jWl!3JVJh40hloXj-x6>^XZ+1xdDZ0$V?usAqF*jz7%Tvt5S{BuNY$zL9;$ zcF!E4=Ed;-4|jb8$vYc8+M$Vp>#qo77TM9QOrudij;y=nga8Sw4OfnD9?xy9@q6Z~ zbqk)~DSy~o4*?Jg&2{+Wc`{_kvgOQcET7(*T)1-n9ki>x`{yyU#-MTK_sk1=`PVK` zq>)_i#zd=cZ=MR3D6qlDT2{}dIZ2)#t^7Rxx>?^&Bo$HsaU^@UP+#mW?fR|3_O(N5 z$wfpF2y8$W4)yaV^jMN17DVp4pU#*Fo+i>yoKGEwNb;s%!j#)N@g&qWtJQ!s!&axF z0>~dEO6P&)xSwtL_hzNmY+0@3zS9Z7)x>C>Z_?=8+?ph5el^4)GDI@FF{M8bSi( zC&N0V;?Tb9t~1bF+T}*Djuqs7FJp$P=SOrPCN5xMUSKaJDKbJ!k?RxJ?lS7|ux^CtBlRP-Oc_?AqbY@I4# zewaVCEh;&%duv2bf#rQ zb=%<5p9$ymFOIYC#Sf1i-r1iJ2Zr?u8amM&5+oXf3`gfjSjnx(pLJ|8=~~Sd$PLJn z$bQXo9$|qrN7OaKhrjb)Vmaq%ut@RX;5+B4b>io3@*nY_i4x(#$Mg0*GG<8Ro1`HA zFs&#Jli&#ElVEm~Or@lueC#uu-WJGTif4hLJw<@d?FrxRcQ!NErq1MQ>$!e-<>2#g zRUQKs2pIfgOrIos5giV?YN@dl88q}n2|B?M7yL+&#-10DJQ^>+lworAr+Hy%>08~V zgx}|fE|2TBgG{T31NPpr3e`%E?O@bP&+g|t3`LW-mq#F1CTaKC$}rE}PEpr0JYBAD zH#gP%*MLIt4BWJZ6il8s7C@n)hz8`X_nh^R0asU!fE{q1Ry9nZ0FWcd)kYSEQ$!}G zrYgO>pLV;s>nyn(J#*W53fqYgX?(aNi!7FiU?J6>N;KuJX}~MxaE#J-OyXeQCw;t1 zJls1V1G`w0yXsuTNEd6gP?1(BCY#ht8r2$n4BrSiW!B!=yC@adnu`=0MHhA zhANWjoG{P*9dl!F$OqNu6KNhvM9z@3N=ib8hTEdMnG~z_Gis`;ii(O33D%gjS}c<4BPQG} z2sqCft7ioC`)rDOm>WTWt{-?VDk{#ifB}$xY()hSg$pT;I^;Y&IDsSv(}!)IIgi;K z0l023d7UB?PuMla=7mcXHrvAWO3BVbn)gd#vgCP0eotaB+7fD_6ONXI?y1F><&Oq*}syce8Oo7%>Oec3ciAkejpU_p~%Ey^A z9er)N!`z$wSWGeg;0@Fk#5r=iuAQU&tINwAl9(>KGi9wp;n6eRJ3PG$-s3FdbZ&fK zPXyQ2@rzN&g!~833=M@z|9|6CWO)gF;Vfsx#MY| zivuY^sGTau+k88BN)O@g7JOh;i-OB*CGT&BTwWYC8D4 zT4iy;Y=N5SGk$e7-k(-><1J~JfYHOjeOGh*pbR;umo33RSu>{26_piIXMvDa?WV?R z>#?DOr(S2x2?EkkIs}N+c6#En7~N<>zZ`l(S~uNEDruytsM;*&s~;LU5`lOyig;Su zP5SI*2pA@_(CO!@wn9HYxnJKdg`3)o!HLPudx-C6oO!UYvpW3!iA*yw$C!CyFom=o+qR67JMHjOx@jYZA!`+9i{OGn#`Q{=zcyrIUncpTO&XOZUa(j|H8`doFK1zlV;Wp0e@My zuC%Ss*4m+1bS~EEl7FKAY1+H)2YBp`8R`H+=i8{AKITTGPQ7i=L%HMW<%znON zIdwK!xv>a$r)!*)LDxdsfSROrmz@dPMY$Y}JPxm*{aD3Sd%i7v(Q0WdtsX?r;rqfN zEG)d{JTI#Z#fdIzxYA&1z0s8s5D)-P+Do$DvG@X*1+c0;{bY!iTq1CC)!SRfb{U8@ z)zNXc8DtoFmz%NwAUfX^eUU7ax7dhhFa`>e$?IO=l`+@J=%JoBr=1vRv2oVTGg)Qw zSxP72S~}l${P+Jhs|5Up{9tbSLtpwsf*Lh`I&n-^*`OVv(ev2qEv%^~2ZTYO#5K3&A4b7N4l-87H^G5{2lhK)4~McMIDEsTNb&F? zWtfkV-?y$j9k^N0R0S2uN|@#CQDFiyuu5*NQF9c@uTyD%43P#=C^I@ZP%?k5g(5Qb zm=d8>P=!?eaCLgDT2x_t+q7;W6e1IIN#oUoCz2{|6qT^}S4H;b6d5;L_1-2R{V$HC z*JQMLjxmXGU7uRRoA3BD5^7C?WUGwV__Km)h=L$dOjH`mB_z{klqJb&`6yvkxoNq^ z-9iJJ(cVR)?qGw$X}L5y?5bEXtwN*DV74dxxQM3NcqZy6Q`l}CYWjMU6)D)r$cAwg zZIkX$e*hb^i8I#KqR;s20fRf-A+jQ_9sQGNf;AcZoW6teTcv~d>6!8%tiqB;t0B8E zF)`tyQ=xbG2&*BvPGyA7mof+IYlj(L7}CX>_=t=YBk=^~MnYN>>wNr=x--h!0v)MV z^puXL$Azh^IKGgn>?>V6IEKcP&n=K_E29Pebz*tYnoN}Qzmx9}7zI-^0>fK)nfTj+ z&*b%k4h`WR`O@~@2o_sbdj0r;D4ryzZ9cedub`q540P|Gxk`WRk-mvF+osiNV>sy# z%!oDB1X`7Vk`%atnXkzvq8!#*uxFfRkQSlpyGUlZ)|Uitmg&4F*ikXL%x7|c)Ymgw z85kYdWr<7qy12SF7!5~u@d{K`!p45_5uw$NZ}?1S{B+j|%zWwHlvh`1&r1pb)GMm@#`ykeQW^5Lq#H8|Jm<68;&R#B|`L|<|kpJ3w z@aH}3jkeo3P&ysb<`*~d>^7tFmU@`3Uh_OgElEtT3}hl!n~6ry%qPXDFbM)0DF?N7 z9js8AR+k^?W||G~GqztXg@TS!Bvo?N z(&h6f8qF3at79VBC3{ZYU0_mkkFbELD7tMLb>;5dE*Q9FnF^DzN46L6X)H8E3dydI zZB>;i>F9ElQVR?|PN>z4dH&frcHDOGYS0sIHk5Or_4T_;)GG-5p=$)KKw@bXx7W=Qp52YiMv}k)7c{P^DOysxB>S<1lWg6;*`! zEKJdY_2rXE-MXeH>@iJPY1txb*a1anYs#Qkv~_qwXEV&%e-FPgt`N$geTklvqp2 zI(aLrh_PT|xbEo*tPBN_rBkaXE7TFfAkqd?T;L=9|8%q$#ppAyb_y^Ai`_jlsN@Vw ztDCgCWr_(3&i;J)X>suxvfxu{tyU82&uwQe2b@^Sk$gp6WpM_q%z@u%fzg0Pi2vK0 zF9IG%UxVocSxUr5Uby_XSD!=M&+*XB0 zT%=}iv;;?WEvirXeDm;Uy%wKKp{}fGWM9*%i|2Ie&R)G`>vjBGR-r!>D1{Bm_#VJo zFIVI&qK}VyVl1Vphaa=sjg<_Rff-xW+B~HkCa6tozr8NCuJ$T--rGrJf{l_YSw5jq z))St0sjxbfYoAqIEI&+LIxO;i8=xF5xBQ!Yuc4%s+8n)@j#r8k1Oy1Di08IspUX#x zNo|JaK~pMCstPGh57Hl|=;|*#*>srAPKUZX9%agG`G_H~FrNd{+zI|9_g0o=3qO*Y zaHjC&$YBDHr7N$v9J z4B{f$oq^P>iQ&-=vq0MN2Acl+Dc6QjUFPCUl5{YM=|X=X#}-fiE%sqd+X10QbNv0< zyJ4=U8Df|`K@q);u5+J^WY{szf`;EFwB{xQbggjr&4}v!W!N|+0v1I{VCE0kVi8B) z_zqWEQ%ciII_-lh*;smWL(gWXt?wrTdQprN72}H)`gCif&1Lo6%ICsaL><)>?e^d1 zU%ZGElqiD-9TW-ADn<-eDM}&^>?PK7i8^Q#_CBSW&aSV^0$KO<;Y@E|cFfIZOIzCm zvH)wC|E~qG(tpN@!k{f|Xh{4ao#uGI?l!aLD9$OxfR%l#c~DWIq@lsF(d8>-H8qC&SP{wyHmR>;oTn@t zga}r~0pcZ3O~g`~1Yl4NU>qhxEwh<&1>d-(o(^1=_tBNj@MLl4MfiQus(*ysbbMVF z6;=8TAYp&3Wzj=MM#O;8H546vEf$$0)vX&+rlzF4yu1Jv3RIL=8Wl zK3YCbX7B`OVBWLtxu#}>!>l2j;KdkkSKX_m-k)`; z#RK`v73d2P8t-&euY2~Pz8`Mm_TbC$A-{l20j?qGG0z>;s=jy3Jbw30HvdGv6VbPq z>acdZb)K~jmX80Pngj^Do~DVp)PeUJ?YkH5cYVY$uk%4V>fR24hzv1)N36rfnf$x^ z0(is(&x@R8<>d852~gs3VFeW{>&i14cBr7SF$pm7R!M_BJvJ6$2h0JC3~FSH;?I$D zdVBHEp~D1nk;dJUAikR^TreNL;0=2c9h1OEYBrhEAp(_BXmQEgX3pdUaTubbv$G$y zwQ<1QQy#muw^-(x21rY*r~MY1i5-Aj0Qy+Q$Hz@lEBA8lJAu>Z*R(iT7^A&XR~EPd zvFdIs&uwgzy7Nkl$FnB&+M!)`6B{ORfqOGV`UF zGz=r__2^M5F}SQo#eQFS)?nBIlbiY0iMV6MZZuV7qgYboJjMHYq8hP|yu9zQ*7g)LSUyY=I$KpO?CM zu&r9IOI9W*EQdvS9rptE-%9*`3&`WH@VfqH`nmm)oW9;^iA|6D=3_;wHlDW+e+_zX zf8js`WfP9mZ(o1@hsu}-s_g8Q#u%SY{XG+NtVZG(PSY)&kE9bb;`L9foQ_9WL9W<2 zhciegY9FG;UGA?Hit-EO*!KVL8)XFB6yGYOibq%$ ze&Y^wT2;2l!N;uKQj(1J@#@1U`q63<_is~V90SA?%|zxRlL8uK+C<3UcDfIHZ8^nP zCx@?_f!$Wn4pRfJF*n(=F4<%U&bzuz9~W7@yxBtyw_H1~d&$QjGeDFlS4_shK?*gL zAfe2t(pLJM=sl?2oQv3H$4T#Vh(zE0fMbQv)~N;CepVD^uWNAVV)v2HihtGj5-TB4 zjEzQLQWosmQQOZk2gZVc2r9dUTY1RsEQ6PknN<3PZ~IvHzW^#hmfzgG<9&|ZlzMP# zN=E~?`=F?hoIpUjpLV3kQ~)hm@?q@ZFHOpTyVth?!+6h4U6?$5K7c?V&gf50P9_Xa zZ+?|9D0X-Tx}?o&0rywaTcx!@T|8h}Yyi4$R3(UU@!xRSHfB~sh);jjGmh;G-_ z6eI3QID2VmTX7@vmM<~e@ofHbw)vXffcOdwvE1^MI%{1yaLrFo^Y&pV_^xg?SYgS5 z2a8`C%WP6ka|5j?{!Wo{RNBfPUD}|)$MEm=nDhil$P!DlazAeu)&y4B(PL*RqQfsj z^GFDXKN^K)oIph)s*)Qn@^{gR5xf#5F!3@2$DdS_5QPD2Ix~)P6qd9U<~*kl~jLZ|y!|dkcVJf!-DHk|#GO*IViLZu5Aii?*v@z~pJj zj(T+~vaXjSPV)5^RnzLWdNP_nqvouP|F8|TAPr0s=DkjSSyWzQb|7u5+jr1 z$r_wsuzf1ZPwr@M2EAWX`o0WRtMJ|ERS3(8ROAWM^N-Bla1nIq15xDP{>4X|;lHN| zL%{EAsfrRNR$pOnC~!c&-*Ee8+bRyrjaWxb1w0i*7~Jk-G#OPW(}Yn@}_guIs73`+|+G>a{!kPP*f;VN5ob@k^@A9)Dk;1eZA5(H5s^>*M>8{cZ4HLn)l<-af5 zROM(#`xfXbqp9eU@;XPW=dKI-RE6dau5!+@325LMj4E%EC@Kr+<_Gl`2qqp#KFPla zZYPU19MXimfe!EaCnjdF;M0mZUw><7AK?(wZ5Z57e=B4|YcP`h2jek6uf-s>))-dG)|{tuU*% ziRF2c_|u;N8&bRY9$>UPDaWVlUwA|cCRBAGejq<15k~;u`n`!4&Qi#gMj`TC3jPLg zuaj~)qR&Dw{|Xx^WceX%0-TcW?wCgNY5R_2wCziVRMUFL#qS678hzucl|SmPg^B9E zpGa)H`nn%b-jpO=u-MLYB~1Y%%TahG=;kO(lNLH|i#MF0iLsJr1pl3jkH#loz9=CG z{J9WZ^~8-$b2GYkHO7d0Fs%AXIBVRgfNvXJIDUL9G;z(zxb81hvxE0FF`8dn+F&K$ zYK~2!1`ivQut%O5);`gC+2a+PZ>=kC`qKq$j<;Xcx%<&}-6eFL|B>r8%x;HQ`@3d0 zg~Ul#7)J;9l&t5++HUt$5?*APSny8KW`U;_r72&(tks13egd!e`?t7DAseEsw1;&4 zs`)>wLm9y2r=Qw_qCckW{7mThy?5~Jc;19J962efDGvq_X^iUK1MH>M5CKdpALnCs z0aO>dHp7bx#rP!kt3fHr(DNPYi6=hupU}D;7{lut?#qv&vkEMjnUz>>V_H0>xU+bV z7Zi-BK0}?Ggd2^R)U#1Gn=qwsWqQ;&k>~xFDm#>$!Ic*16}kMc9#zi4A{W|29ljyh zR^JwvaOY3K_&&7#x}MCrb6zKvk{}UW-MZPo_T-Va z*0tOcW13N{WVut11!z(3?$}!RTVqE3z)qQ&a^R6#J@ey+{i6<@KL?PJfvU>fC=?!d zmN&}eB?!qn%SFX)7NDSi>p9{wwolw_NK#JQI`QGZ2uZA>n!*b$w$HFCKS3QCXojVr z8(VM6ym2;zj2&wL=`3J?;EKxI0=IcJb=1Fu%};)%`wg#yiBcl5$y!D`-)g>-UUuFD zDq@d#PfjVZww7ZpMf!31;++$CagR)96-SxgJLB6Ox0U5No|wAV<*+^(e%0ac``-#0 ze}DWuEG)5LlwM*p`oaK=719=0H(dOk;@#X6` z3hP&1h`OciRrg4l(KoxecyCAPA2=(cn$Kh({J_jRH0OV1)PK6VFsBV*Y;nM@EI4y) z2lNujDS)04K&)#u2hRXJyY~H>A3SnW>+lZEC$X5#skHpqC_P$HiA&o}`b%EF@aDIB z915hBE$>JTdr6+4;{=&f)+PzWzp&0Ufn}w{n*~7%@J)bj$?G{GddHcoyrN>@*tLiz zH_X@o;(WapiFt0>_G_?g$30Jdk{6h6CHx>GgY?gf?>~Ie!|xk{tY{WPt@42cFpLIJ zu2@=9B7_M;Nr95$fq+Y3SHruoFe5Hw-QZgQp2tI*(hMK<3N9#FVwlh25F$ubP1*UP zvSS~nP-3yB$@cAcxLX6Q6=uRLJMasREdg^|8bL>$j8cT)&Y%s**rzyjhQawxQ79A6 zyeuGN>qGMtRzk~RglkuH4nol}-P#+h&YPq2;PrWF9c+LpnZ$@^#MfrSl0p$~JwX`3 zx;GLJ51~yd1SDJEZ1)+x;2a^hC-CPAdk<-aTijnm8ZeV8$s)x`?rjj%tTKBqY(STu z`mG+?RKoldj*VBY#v%3c@z3xcC@P1?aoegMF|UZPaeu9z5s(76*7Sv4CO1096M1}0 zE2&P{*9J=YuW?^?-ygUKwi({RMc*16_o>c-OOfU5>QiTA;nwxzN|z~hNQmgmF8>Sf zXxujd-PmcUC2bF&fK^*xn~hE^Emf zE7%>cbv0(@FsnIM0Qd)xST1rERQ#<%MEeNPdW1+=n&cH!<4@>qMT)1wy~KK^BzRN< zOp+}*LM|R)mbK`=_EQ|sfGpBnf^2NhQJYd)M=c)EQcb{9j}5yD0Tt^-)`@gR6Y}99 zTzdUAu)g_i0R!WzZQiC7>~$l(Hd!Fz^_%81QYQOU=i(5HQ3PlCVR(vi5{Zx*l(Xdk zU{$!g(A28PG&*1gJ&d1K(vPtc-k5Ki&q=W_$Y&d&&FzVDuVOq@&t+hufE7XUc}8}{ zDJ@Ok`I`kspA?F|LXpY#vl&)D}e`@tag?}~zlZF-*sx zxZ!?T4_6nvj{cxe>9N(SG!>sfmpS{i0z~fIEptU_tVt>d6%PU5T?90z^VWyB*B3=D zZf<~t?vZ=#f@Go7avsKgH=R(w1k3aNEKefWXtm|-aVw%MM_Tsx48f%)vbHYBFTcq3 zL2lvQL2C>2LdUeGslC;j22Psg0{(}|JGPsN zK=r2pA|?R9De37UXkGmp8ryJ8#wl(9tbL~xCAR2MHlQzN8@B73LvF(DM5e-S76jX8) zR+M0=Qj22D+1EJQPeQWBOot=fdILJ$zea9P3B`49OzIVJe?E+?V=oMmv~tTTW)#?ikxcs|-&=F~ z$O>pmC8Fub(jR0q9_rY@I=kITfYM7;2R5UJ2W0Opci!$oc@iE^U7nAZ>vgx^JNw%U z{d?vXI#+|Ls_8^(HT+k^?c2wi>+?A>^tfM3u92xG*wi>w4%lgyOQ)Z!!|v;P2m3p; zmlSpwYbur3*pfuTo-H_60-fGq57667Gw{eIg1wF`F334Mj#XHoAxY8EFx}fyf3L}n z9<`8UfAy63kv}V=e8`rb$}O#NGh}r0Lw4kDq+&vT_@KH>yVlNsNCtj>-8FXJT#J#{ zltyPkz#|~eGIdaF8DmT%xZut9kJc)Tw%$Rlj%fe(SN*bEt^705s zI1C-|vkMs@l?3!hjauZ^t7HIB@PvAuY6__0>d4HhJlD&#*|t2W0@pnQO`74*Ai2@W zUH{So(2BXsZt|k|RSX~v0-i5-4U9};kxemj!`?=!$o`mfBnQEMaYY5?`<^eRV@8S!!Yd~=Ls^PO!K;mG?SYtz&= z!Aq+jIswPzLjFCH|82DNST=@5)h(@dtM&b_mG|59;}VU8QE4dW%^Kq4Thb_{ZbNcl z{FHGo*YK-q^P1}3ut&tJ7tP!h|JlZrJU0nQd_!Qt`U&m5=>;PdDrkxJgDH8(%I&h- zeinE=Q*>*ld)KFGOkTQu>wMVA(qwY{PPGFt%yprRNMA&Rpq{bM;PLjQrzj%Aqe7s0HW7PsfWW~-^O5u00>9OT~N+EWh;vI&e(3HjZ zVSX}0+4q&nStN3v+jpqd_Yx(_@7eF_GAxbQ6XELY{SDgtb=5ZVeGiyI?IrG%K|e zppz$9s9TRs5_>`LBlVj6=>wfg7;aIWx7g4AeMe-N&sP8@6p{f- zQ?!K>!&&ZJw{1U9lxaZ)FrkcSq)&!f{sM}uDyqmy5xoH$SAy@ROL}H%q8wU6;;ru&iXY~q7T3a=Ml(;K@5VkilYq#B_zh$2Boqn zLC8vjjMft!vEUDaU9^(KM#dcj7TJB5Vu~zZ*AHgQc{Z?%@g8GRO`-S)WgpN&>7 za(i4wC2wMnDn~4pcH=f>}=cHnBe z7dXG`#op4UCK~e5D$Y+d8gA@@tOL|DJd&gPnK;a1^XF%$oK{N8`os25T!3yNhZYH+9?RF6&dPeXF@?8ylSUx=Vj;ups?TN)lDqGW z#yM%xY`5C#M$0eN!yP)Dq(Z*j#!4>)i%c<7)BUQ8P6jq|Ko%Oxn)E5^FJoaA zxT`HE+|bC&1?f{`Z_+Y&-92qR@|=Fs3LCg38L)uHkSYI>vvJ6nRT~*E&mKNiRiJG_ zWS#}6tYw6N*w6^y|AS{>pdsQGVfNJ(wHtaS9xqbQT>MG`LY+AU%frDkOfja>pJ}Jg ztsOj-B|rNV^KeKj;=*wyHb=tF!rQJLfi_6H?tIEO{c9!mgdqJ>-Q?$%6et;}XiUYX zB>?ploo5h&3{I^swm;+2`LWM=!KH?meetT;n5u3(?#-^vULM(>)aSl&@=zNJUH6RY%( zyB=sZho60DK}|;JA?)kXmBlhp(jbp|Nh=GBt!w=kCcmo`{`}6)sKY!jV0yjn4!iG3 zV+MO&dxQILm*@%J3~HLEEZ2tDlt!AK>mV$jQBKFVb?tN9&;YJ8GH~Eg`;z+&CjE8l zto%>TzTc~M#*#je1g~0uEVByE8@s@N^^(Uo3#gE=Xd%oP%x!r?Y@^l>wJpox8W;xe zVONXGpYh_dt$xtdBuXsS{b&d+)PQG2>Z-X&-Zt(vI-Etwj^Z0yD@;(VL?9=HOH;zz z7&-BZyDZSgEx)3|9AjH?2^a5qzv$Y~9lUrSCf86p7=P~ALWuyihL7&TfBibUxhZdG zNF*vM+Q_DXf)EC@S2T8THrlM-+~4oob5PLGWH(;}I{MkgMNtnAj{}#Iib5K@Ax^yh zjaS%NF5GPu!UE1bcSE$e8IwAr=Ui?tDkA1B>oVy#1{5NG z{>vRVWxEY^=w_b0-A;m^@=sp$*!hlY!b3MatT9`LYTSHGVZiS!2VX=k&uFVpd$Rpq z_%jB*=wb}sl2yVj_K;U7$MvMGCqgz6zAPclvo3)#PSTqszoc^$DxXVs{~&aE$HhmU zMOOBn`p&kd9Eq=^9KE|As%r^41fRZ}{$64(#p1cdP`&pH7uZEn(e2qGZI9Wuw;z4p z7!7zuyt>$YQ-6AXe(1k{%ORCct3Up|YPJwc%IchH6W1NxeDm>K$Mo0ByxaXVy>ue+ z*h&zzWgO`-(AVO2$2%tP+m#!~|I*3hyh&CeG;Ni1OE?FApR=`>0aFk{>aJ6S05?Cx zn_L1ib)cIY%_yD{dfl1B8gtGj>3)S@rPR`j1%G!!otSRAf|NdNk@rDUH{ExEO-^sh z<`5L;kQ21z6g9?m79HKM%>T(PGGFc_GC+B$w-sE!o4s~Rv#Q4_20o}|h;zL=<&Y7& zR#xovfBI{*)&cJ*4N9!kAl1T{;kky7<9nijfcQ6Zk?;L+rqOb~L{rlfI3M8}G-8&P z$5p>0dV2V}^e(Cp5}Ypo!VFzamU9{j=w!>ns>Bb7H<7`NLw+I8A)JK#XT*`d$~oF3WXHt z?B>Gx;kQXwB@+E7jyKp`&c*iVyPA*e+{@f*Q`aSi8Ba)s<-^2;LkO(GSNsq+ZGD?L zFvn~zLE<*UHtMLwrM>w{Xoo%Tc5%beav zW#WVPJw(3d1fgA=stD_-2Lo;&+7ylbo$r?X@q+ z_qEl-bkf5f6Rkc*!Vr zy!29N-3avJh+cPyG)aLdpoEzK`FipL;i%J=GSY1CHaq=`W)+>Vs45x_k%VU2Qj&=b z6v~*$ze+YsFQVw3i9XS&8-m5oFG<3RKC(nnfjtdCtidTMg`KuC!sOIBWPA()o8?JO z-v>yLmsLljS!CSFR6=x+Qi^!xdiY^)OS5Y*ehS`0*b%l+~t7G<3yw&##a! zP7Y76W$XfB!8QqzV%Xnpe%Ng45$o#0f8!LnkHt*C?vKm&@&R<-pPhTxMNX!L)4^y6 zwQ;~_rpWC5;3nArb!-hu$Y4bL*5!TmCXZYC)SDT9hA*bE7+yX0&NW_wtiiw#BJXwW zkaOKR^7>GF8GIG>vgG#`9oBnE%H!ceNH~u`BYDz~-{r1znHy(ULU$A#u(nC?S*lV~ zNoy*R6?esR>ABr!@c!Yidhb*c{}aK9o#wF5*QGu=xPTm)1PmO6uvoLKqtVzMfWuZZ zCx;2>8yRo#Q&kNbHy}#s7y7L}y`B4)vx@75?cM$_{5yET)%J>$(s#@E^eWN{kD&!< z7J(@+;K)e|+%;l>n{8pbFO!1*>Ud{@T0a1_;dJ=Ql5yU9Dw-lHe^ zy3{j}i6V>x6(QoNdLq2qGWtGcCZ1nNah2SbJP6{Hi7F)glamF0;R#${6s%bLSP_^L zjmm#ZRxuw``Q1>DKmvC@BqJY@PFGa555+($FxiGAeFIrRHMKO=WDOEXcdW)FN|?A> zEE(M^V&0N$J_LXNUbfiH{v%3{_MN;6@W#bjkF){rZ(1#Bmkt&$>ngP3QNabY`X;vjqrUK-jT`U8*7f z^$Ui*D&$PVAC%NXb_967F4)7b@0Wa^l2xhJ4Dt=~BDVk(ig}1`6ydLh!v@!!0|2w> ze8VuEsMJoBxW@z7jpSChUT>B154OJTTLljq>bd)JU-Z6z#GuhhU$TnTxR-3I_J|IoA=Zc=3_BGKYeqvD_Es{4+rWy`F;cGUbJPgy@J=EpB`NkoU zB24NrhFX&9D_tD>pqE;_961&qo)~q`Xyy+%in}9< zB_%b&EQ#kaH-F^`v`~baYFcs zpK!6VvNpERPYrnhqqO~3&vG}usiYKcHlttc$`d_>Nn&yeAZyGL{YNOSaY$8MQz;Zu zzyke}kA5(gXlXUE=Pu&0<)HcbiFn0-JsVoix@lgfVf6|rOBoR4cJOIYiJ@Xp+L_oKZn4XcW;>m4iHnRRS~W&N^)V4I6i?yt(FWw2ewo0!b@%Xb*1bHdV0Wb6GlIEED1Ek{Bx%!ID#KF*SAcxG*Te6vbY{NAPrG?rw94@UNtLMdT4M&0Ufb zD?YOHO2W)r9x()Z3f{B$P~Mu8Zd{w@P_4FS{3;{ly?3Nvwl91rO6^3QFza`GwA1g8 zu;<1=OGAGtxi$uH#^J1?*oACJizU9E-H^mTeo>RY|5Y z?9p$fMjGYXwbsp9-jIX|{A1Qiyk%W`o_a8xL3DF&>b zP1aE^0p{3E$pMVppNkah!cE~K-J3Wke+hgB2nQfwgglE4ZYAZtfLhmGasPwIW&gpd zr{bTYp~cOFEDNZ8Z76|x#c|_;^G(MSf{F%utRtyY?zw)k;Rz$KIE0@M1lR$lLjj-e z#z0aJQi??u7$=sIAZcV#Mnyu)9T04343{pqsHD53sHzIoPD!@_PR`U>H>M&sDpeBR zbo+4&0ey>>5>j-me8EJW(7(1SyZ^wf%AF@J5VSI3GwcZ^DZW$I5nEdPxs*TN>3Az( zoC-cJ%$Y{4uo9ISWvri2(iL0$k7vDDkY7&&lXEF^sY%s$97xvYrt_7U92pcV?zkMe z&1FJK3~uBT0ua7z{$H+*Ih1T)zw=|OMLg%$EN33M!b%8cjDMu}22zaZ7j(3%E#cnj zTA!O;pyT(Yhe(t}r9JxDZ}+-NR_-Tzcx(Ojm_7=Anc+nDeVx%`m-i)2e&-q*W~#EX z_6RGbLOxb7EUpF{dmp|0DX~@q$~Nv7kf;&klXV#ljbRUCL3YZ2)PB{|BZr3=x_?Wn^F)ves0%3SJiHRD*^(CsSvCUj=wiB62+;Qh$YBnr zbBcO;as>z(toDHxXGt0ih3 zNGQBPWFEUy!tWZ(1YePgJ2E41ye?_G(Wh{?MpT8`;A{IW2Ct^4<$z}Ygm+%an}l4D zF-R6yTc=|y)n+xD*9+r+0(Y8jQV;WSX=DjA%#65jI74GnA8Ka$n^RN9KH7x{UU*u{}umbUWY zXb{oKY30@}GHO=fX_!np9`C%Uh+j_8gVZ+-idfc!16B@^pcFzCS)-hx)D>S$-xV_} z4P6(P-QN=VRJw6dfBZUuFLnf#2AF&u&N2!W8gBLRyppz50)K4oNS^davmf6LX6_Bhjhpa!q`sfy*y?sB$14k_J3Mi9O0>b2Q@|R0rq*b-gmdA}HM~zi*jQBv zm?F?MuL=3X1R;d?o!AM2C!*ot^ z9QYKtYJkjD3~y7sO0M5ER>e%BIgjk%X6v5}>v%eglaVYe0~zFT_)b2DF@LsBcNh#p zWzr$bkgXm5^4`6bhqOyCt*ScWmumGsf@A%q|EnuVAGaNDmEiqWf^tkJr8AQ001367 zYL~{7w}gEB;V*-^i;xWLSZdXewm5R^OXUErVUk1wUZS$4Mkav<99*3;e1w^R0qGx~S*_uH#g7Iyaf zljFQg9c9J~zaxkR=$d{>9j9Wn)aT)xHJq}Y0HG|*X}4KC>0g`+gVNoKsS9kEvrP0{ z@iO<9{2T5mGrWy)Y0x?_hK>hA^Don=y?ed*mc~m0=_PM;1pKQ5{;iRSs_0wF@zH>m_5XYxh)2Q!*Kq}jiY z<&<4rOQxBK^_Os@bi`v#HbA*0mEM#T$=uTDZ+W#!z4k{uxBz?hpYXN6+J=^(R6eDV z5iv*)pAJZNFz{BK zWAq!7oV7$cx5YH@a3y;TbOXyLdx-aKNcg6eT8gV{s!~Q02vj#z*oIv=hN6GqXwbTE z3eDhYMXP3S#j%1#=zGT>k8UQ!%eQqc-|YipF1&%CJ!iem1a`$HV zOab<^Q4oN!ezh)vvq3>c#nJCg3DIZg&^PtusHU#JqLbDTG&+=eLNfWj?p&-E8wOk9+9 z3P0uh*}AI$l~Kq(zh=es5#ChwKU1Ui+DI>-?oW(p#l;byhrX?{noiI^#5D?-;A(zY z9m8X=T<-uXCq0qHVf)`dD@uQ4|3o&Jg321Hm6hRbqK{-jAQ5O1c~$iZ~b5<$Jeny!anKw?8Q`Cn*bli$3;ooHzzYy>5 z9lA11C)EEWqZEV~!#^no1TLdxl-Xh>G&I1fv{`=gy2*bnx`FNoM((=geW<}_uQ2h+%`HiV!P$V6GBi8ef@yYwk?W3@TVc^m zLUiiPwP>U4Td`%Ky7e@vAI=VZ?4y(uVlpS2oA;l%zAyv*(CM+~YmikulAu`2HBwS+ zCHkUsVi}tj(w8V|*i0sBbuTpf^t5|AJDITI%8GGog-{EWlxDS*+`MOcXsL&^%XOQH zpzl#@{9;(zTvXLK+18xK(tVhA?moyk0nsUxGQsTlY(n{T#xyb-c0l##c&_0kkQrE60AoRapj_n?0{Y{fR>@xG8%)q{B z5to2q@O{XqC+XfF)~bwL0)iI7rR%iz;i&2Y1}177R93ETw9sPx#P?E5n&ZhzNaE1X zF*EK|XXx?N3EzY^Q6N$sE8S&9Ga>8YiO1Ss*V{#Z9oSB07@k$BePB+6doeM9GccqO z6a6y-BTmL~Q6Du$DSdtZ@kViD$f(^P%bTbi=BWXjuXDu;#v)&(Cco$HFjhh13$6?c zdm1zPrPhx`IZHQBjo4ev*Xv@J@cH z1AiAVWPN?8ZpI-I|oct52dtvZ%8%v zPX*dqtPYo550?*pP057xH5sB;*k@fAlpgl$C|iWj*>TZ&^1C1gOJ1jnt(Pr_(F31= zs-qeFscfDfa(a3SoQUS;=XWoiv`{{qo3mL!YhL$no&xWnxj9w`&uW|>JqsSt62)6punpex)P0~BFNeLtrg3|p?y z?@}w4xo1&T6rp>*q^6ElDxULf_U;2$Gla|6VAjC(i7N2ZxFL2Q2xAo!SYHjw@>_F z|B%MDFTFqPWwSGBU0vN#qnZhs2-?B*q~2RxD$jY+b-O0gqC1H!6@5y_1`^;LT@DK~ z)A>z-*7Lc&cGR}%xd4a%<6q|78-sw6XN>)knIThv!?*>$iS~zIJgD-Lh$T1tT{mV@9EUdU}0LXmGOW1W*kI+{E6|QS1?+ zOOkD8kDf!Ykk<)ztwQ_OL{U19*ZSO#dkCV{R!^_%uQyWV-H-6JkOr%EYRLw#BlR)o zYj+KECm^KduqFo!9UF9;Wr3qVLu(oQGL6e&zaY8x124!$FRza6nqDtLq3*#JDQc-Xi#y{W|&uWE3!}2K{jm8nL$%sK|z_RRE z<(~`N+`F}QC;FA^EIc^bV}5Thl2d~6HGOqlIq3V_010IpcHiS0G{0Of*)fPmvky)( zi4>#s{glU&iMLUl#!h**X5a@&A>iZXzV7)LmT2S?t^D|4KEuzeU3jRzQhC++h^W*W zrFXlXDcEBg>6K}o>7Eklb!G~@5t*rBI4MD2ELn5=$DMyr2lp>K%NHRHGP>uCay}EOmXBb^Sfz*uE zYMiBK^K=XbtE1Y-+g@uD$wFPEOJ03HwJNmn)GeXWZ)Z%+$nD(bK~@{pt_KwR(Yl z=gWHCu-7`W9`%;zn6yz0#j#$(Bayym;t#j5w%&t97(1uXP^_|`q5f*A_R*Kdh&UWd zgjt2450)9*+ZMwwu$=a`UJj@6^~`J$hIE4kt)x_2cjEA|!ne-*Wwz+_atEyh_bFBm z$VkyEM?v71wJ3g(xEv|W5mD`3AXk~E!oZpw9sLqqH<`B!n=`H6o{U?9JUqS25Btki zu7`%e{Sq8nulrpiV0?F!h2{Mn_#^A;>A3?Nu&ZI$RzM9u90Dh|c+v7_Vkr7BNjV)D zy;}hGcm5G^n!N7Wts%D*q&FOAo)7R3zJ~A9(kbS*;SIz;5y+6=zi#Y{EO5eBd>6-N zea3}zlpM%)iPPVNQ_yLR1>MbNgT9;G7M6TT8^f@(pA#)-i7CSvZQ^c8${6;MB*fk^ zjo2sN?rzY@&1GT0uG#zDzM=Q^a@+&Y$a*)SIo+!tBYBl@;b#z+alA^Mq*N1L4rL`7 z7J>Kjf>M)q{;6h`mNAu|@C8T;*zh~YYlxulC<w_Wr{Q4!=`OJ?BPa^&>5<2gbMw-v#g%uiZ*ZD`;J zE_PDAnJW?5EPos{s(P$}@p^43l6js**Juec-=s1i@}FD_9>nOr(nGGRG#@!9sC=dH zGlV7O4}PmwjTcl{mGt$t)Vq_v3N{P7wKp#n)3gQKIf6RyL!!d5w!H=^t4h#fS{85tRk=YhbjCgc_U3<#|3#jk{M6K-|Z z@4DQHTv3|5m2VwD!dwXC!++5haByz=4q`Ox3@iCFVE*|tAt|ZZNQO|V7|D_D&FZQs zRn#X^f`Pf)_kq|LSXi&n0=M?|K2U{Z2jIx!LIRrmNmmIb$r)Pim^FLi&L$qSx3gspcrX<3HB_DrW|6PO+ z^d--DrUi93>YX8!(ZQC>CT5B27Ylt;!VvZS95LqDE1}5#d+qxpCSP6+?WRJedyT>_ zn$Ti#l#YgK^M;3@mc2Rep+@qK&2()&4wiVmZhhabaO>AIx~j|5i!`4fNP;*+4~BmZ zun-k@=44Ns*Ym4Yzm$T9&W4)SQ2$`pqASQ3Qx(B@@dA3q;z$yldAClFfq^kPH5Itq zDEN4H`l(t6h=t|-u7RnfogLfq&2-(9$qTstAn9;Ba>JqtwxxyU0)??ORnEm^JCSTaE?iAcm#A8zm33ECKBfX^IZt#3tI!Uk9FduAT^{ z8|h<}J6S<@r@b>(N+rMr*r+F^sYxu(TAoTFX)#GEa>Mob@BC#x)w`p_%ftLG^HQ9o zxIgLFh>wSDq`8|Hp4V3ytj*UJH~$i^*9soqEq^&!DL?Q;Zo7t38eUIV%P1!eYh%mH z7cH4=;&2|Ha%D%GBR~ywyO{>;4=Td5%rt(m%^5Mrv!n@zMBJC}0 zjz*GU(W|R93!P;PBhCg!kGLVYN!K6u_9!vTF%&nZ{2TtPpTuM~r_iLA-E&xsXp%3M zxlGlTk20$f9&mm3A><^W-(2)Fh4|%vt3c}sI@zRRDIPZ^+n(O`H$7J3L- zsUfH>xw`x}S2O2{w~5#D^T_hjn6S5apUJ!&{qWLcPh-2ruE(&Xri1|>&wJa%TCb=j z6vnD(QKQ?W>G^qgQT%wWX0Il}7Z>qcUQxnQC>wN#{M~+MisGv0_G>jgZi!JAYdM7< zQFD)6OF?A6f>1ofaj7T2+a_r>i{SYlG}>Ayl>8Ej_uL@x+?TqbhsC=omgNS^7*i(p z;lWL4w}{~N`LEt;?u4nO3ooBBL`$femueB4%KlU}ygD4`ik)32q3<_ydI5YIgd-t9$`R`~lW9^=sz@qTIm)`p4U$;y=opM?!>o6Z_j%8K!26VADl`<$i5CeT0S$uCU=%eCdH z=nEVAT$^EZ{moCcWkQ^zqo?PFw(X5w61oCyvlQNZ!rYsy*+1VT)?La0OA<_fax%)Bm{>^cN)61 zWQY@jU_PAo8~|M&o}QI9sjujj;;e}y`?e1bfFCj^m=CG)y4f*6pBS>+2>yE%N(7K4 zHaRH)jBooQNKHK&*|m;u?}6|vR;l1M5fRgDNl8hV3{58(1e~0lXy;$Nc#)7{%2D48 z9KPZFo`KJixU zv9WnpbvE--{OKF(1F^J5ds|yS!9vBmWv^IJgTKZs)5XdvrSU$;mG}xi8dWq3Sg&C- z0H$&>%F0;a;rxh*NXP)5XnMf3`so)q_-~Xe9N0nB+RXQYnIt(Sg{j7BFyRdfB~Y(u z%VJvsY^0&#sWSsW7BUtY4K5l7JG;tB&#KVb`8m+w^Lm^y6n!Xm#u9MZGxqXInNg}Xd$n}WD5Gn5+V-5&HInd1lbv5(f8#i zo)-Wc^eCl*!HWXZzKA!T8$A!s;GxH`ou8ktoL+-R(fzhx)Z*PG9<>?r~3kxJoce?CHWF*_Ja)^W&Ds3vsQ=5p68b3+H$QFY;yvKds z8vb}{Fn`tic=LSuaY@8@>*)5dA*3~`rJ2H0c%Ud+^)u?k#JHGUA@;+*)dSwe0@V@X z+Ak{{yNYyA+xaf)Wv^uNhds=tB^Oe3b1Vm%*>>dGTH|Ui8KWF}StdoJ4>h8jOm$7o z?Y%^SuV@bs=woF*M*hhuhb(>Q|E6+6w_ax_&c)MQGpW~ut7k{jnIu>GC-tIRbBdM&Iouw?6f;b0QKPJ6_)Y0^#$baR^)U5ghnU<^Shtfut}2 z7IDDqm|zw!5oOH&-DW=TgVD8D#uqMK4=lxrFJRLLYy!#9e6vV#c6Cks=0g^VJYO5q zx0V;n3Tz~SE4AAA{JdV{H1NnKMB3lq2c%~Fw+HZ5gHrgS(#5 zMikB07oYM?Ly9E#1| zebEp`4;tXtGT-*AcjexC%gD2R_=By}|Cv%CGAUDP(EQ7GQ-+&hYiF(e{=d`f3YIW% z={PnlR6TsKhI!n}!XQtN76Fbo!oP1#=kt7Ct2XrM9D-ra+uNJg6A|vY^}ONq2xb?! zafyhCDp*RaeVOKV!Evl)W7$_0vs4Hd_@e7ErpcexOIYdl5yKjkrdbyY-?=I?^alMG zz{@|li}@zjOb4_s*&=wi-%U=lC2%B#$_RA0vDyCprq~|O3_10aUOaQ!4}W+3cm;Vl zJ0xH6T)sul#lGucTU7A9x?>gJ=Koa1=cw~#d&ksH9z|T>(MJqzls4tpOx3B)!o)q* zviAzDkBbfGFtPMlWbQsQ3d0~%OIu36dV^d`dVOZK4+?;gf&yEoksIR( zP1h)Bu1W1g85lDt+)qsqV(?GqYpF^Gdac4=^`k+HzP09mzzNI_pmR!Upu}*MJI9n< z{8&GsJ!^`>89FKW1h*kMUF_*ZkP7rZohh}WU-hr_5J4y?JGH zs|^hfqgA8{yJL%$;)MQ~)KAkU=`V={opCuE#j-fx>^xHq`GZb`SFH$3wdl&_uY36J9&V??|@ zR!2x_IzD&Jesiqc#V(gEPJ3T8?g>2b!MzXgT>#oMhZMzM(Er?f`cI~A)cHJHrXJQ& zWbJK*UevsK*pUnxM;lnP2!bZ->Fq+W#nl(qVqk5dYA=8k}l7fsH{?@Z$6;OZB1`4q-@a zlrKGPbx3{f7bq*i&;4YO4;%K8$Zn?xy)Nku;U@48?`7TzGxGRJsr|a1b4gUw)T;23 zZL*|>YWCJ~Yrkht$dLAcBa(VHZDlim5kXn2p!om#Luc`HDyRSMl7=vyua{18Q*!U-|^&}*H z7upm-{%{nmpr8;3g&5SuMdT+5Z;2&Lk7|41NE}b=Xlpa%DHdx5VE-8zfqAEZ@d7lB zdYYT7RWVLcl{f7vZLSeqgIOS$d*XzS#6&jpgdkd|Ju9vNyk6t@` zOBV-5oB8!EZ~y94ze<&JsVhmpPW-&wlS0GnLoM6DS|CA|`A&g1Rd0S+4L`I&K4gJM zn^(0e!uxFd^`5viJI+LWI)z0N6yi@9Oc!uo=lESkP#K*~BO; zEa=|xe#Jl-HZs!SN%)YFpd8|XCRyYA@P(=&TNy>@tLY`>4-}>^UVPH(Hm_#2a*D$; zt@t~?N7dCCZz$yq<`uXiq$v11i>k|u1e)Xk{yT?6Vo&O%^e|4@W@WRtm+9XaRH|GAVm<_{9p z9-m}}y#MNDmE~5#C#*Uaf}{8QL@Xq3i!+Q~Cf}S%P;^|vb~pQd)nu)P`I_9F#?y0s z^CK*9Qk)=nZd_1iC>TcuH+6UoLauh{?6 zv))s*gBhb--`L9v%R>wKqCvg><0}g&u9)+DaGfc(V%yt?A`7CiFLCy~&3t#F@XLYQ zwIr&dU&Pqbx6})6a0h$k&44?MqJ>T!ovcIf)oR2mfzj?V*gJ>ab346jb1@v7+0Oe* z+m(l@CARS%rRhkak&cSP5r$2DNeik1; z-@;@iI4O3MU1Qz6;48uY-e>py3cRsU76N-d&Wv z?pYG>EaH8mH9fSAH0eeniwSWt4GnzO^Gkx?qD{nA0Nfd&O}($)@CuWdL1rsOlx9I@ z{jLz>Yig?b7iTosMRY1}_XA&;QDa{G&sN=^)lxZ`yR+(QE%dSyn7M4a6RpgM?_VCp zlFz7JVHd7T419n~HC;!lrJ!2*_TF)QMiPxiZy25C-<**mPXg0o0tlO7z(or*so5?Y z=7#BaftLnI5=#sxU@&RW14DOL&pk9K4w%@%m=4h2u`j=DZ72eaO-#PMr%-`W$%0Td z06Kt`UuM^oPoCsbTkr_Wf?^NnTC$#-1MfVIISM181g+2+HPyFZ90SP1n>TM#R~v{@ zf?$)f|Ma7xpg--K;$n`E9Y8cjmWoXHP$mRkq29 ze{q>+5cecCe8<&;zu#21E<3rIdMEXPB40BvhsjSOtA4V_R5-Iu?DVTVc{xd+C@j3H z$&|bOS1Gsaa3eHqm$&4F1pwf5ri;U*8EK(we-o}T2F#qeP5RD3dk2HS6kt?OXsO1b zPEHiEzTTQUXrt{HH8O*tyV?nP{KaAp@r=`Mg$VJd4@~7z0qv08rl@K_%SbH3J_BO0 z@gP1ul@Q6f?&Gg_A9;yxA~TIyKT*L2s#yE%ki1QNKIqhQ8xZHH{vEy%X{qPKXPkb1 zIIYsh9|o|pTj4D25P*)KUyR9IZ{=&dAgWQZ-F{_6l@M^-gf0fe8rn8L;0xP}i-ZSB zKzRyb))gOHemaRJ#b{hn(gqfuP(*$@Z+|$^6(E??M!H)^x~Y8bWa4=LiZ)iAP(9@L zoeIoh)xq)J8>P*6Xo9usvtMB#kkNx)Zy4bBl*3mQK<%Tj@Qb?pll&Hvp)5yKe$k@_d5B$axc49Pti0S zz12XuJ(+XMGeU36)b}-7f&VC!m8<5t#jhkNWh+nKc{ra0D@D_r7HfP{LoAh5k2?-J zh1N%njo5*dHiB}Pyq)X2*4l)sUE98B1$sKS^Y}!~MOhd?iZXVgUFL~czc)Eq3nIw> z^+$K5&an-r5dpsFnYWNZk{MdL9BT3;%dL@jpaPxx1^1obGmu!-X(jnvI8uMuY;WPU z&%^I9Gf6;IZtWa=ysYv3Kq3i_G!t-TusEJHyAK`wZ`fA7q<;wpZ@pUUj@@&^bR_56 z_v#o{t6WO4$QEDz@~_heRhp1~>5WGn;o;#inZL+?iRz3gxB&t*-?~33p71tELj=YT*15 zUl^C^rQCE{9?MG^xYpm++KN1e>NN2L7N;dsKHTHXxWrXk z=FKMF0}Rl5u>I|?D)je?Xw9{O%ZEl9G+?xu$&WCL!(yGyj~Lkm8BK}z_nn=c`A3LA z`p{Qffq^-}Kp!JxJqSEv%89-nGhwtqu`J$4zFM7e zG=tw>bc7hwH7W>*l?)^=OjEQNF?KGqZMJ)TE^sZhzGYG7QM}{(G177z`kkhyp${G?md zl!X(c)q5~vdW|kug-f<8L(2<3_)2&(7v+Y+$@*e-ZZ>_VTCGB}(FsY?4S!%#VG0qe zZ|ZR@F?%#VbC83HyeTHfS?RnyJB zZObDzb&I`#Ag@?;-F8lqCiiw$N1>G@F4v3&E7d@JfYO40T&7nC)zPx#j2-+3kUa!|76J?dP<{Z#e42Mb;Y2Cb zU^{iJ${}AXU^Av6O-sU5qf)D`Kb_bbt7XQXW58u0Cnw(o*>sD7+_I(UN-?F>G4;^F zu^efD2>^$Ll7{&IQG8N)9UhS4mESY>o$_%*Y(uS%kT%1e`4Wmd#H{WFoa*s8>b%*I zh*x1TySt)ScY&7)1RvLm6&f_;ZJ_5IVp)Z(fV{7)j0UM_#S1OI0F{m)6AFdyO5}`b z_>)<^@!?jaCd7aeGidK}V|0PQyhJ=CwW7&<9cLacF6jWFn^k8CQ zg0eP6a1Mv>2;KP!-WXL%=q^;3xj4g~1i8;)XAVhC?&SJzsw{YRB8V))SW$sM0_XjD z^*a5CiNW(|1T^k#0I|?9EkNVZ>cm%Q20W-jc3y=}J=ORn;n!NT_ixxnWv3Z3e7IAM zN_fmMo7k@HH0GK$C2$(#NHH7qpXrbELYJj$>xlIUoILVR5M|h9YF!6VW5U?=AVRJU zH@vF9^5QfEq$qN6WUONB|$sbUx8r<9UB3six_4ldXHa4-ss`A(kH{LEblU)j%OgyVI*q zjMJcdrs;7~B-e)mPTNzPA!!jFJ+fJ8)5LlJW5-y!UAn9?A0g}Q>$9-2$pe(Lt-9j% zy$_e7I=Ae%j^eUvmptIO2qLZ^w?GvJeg22Qs55PFjSFARw&9esegNUK_^Fv4;1>eA zR)#cUyBn`!X}}2w9e=h0fd-w*m~a&f zE)*z39!=wsRs0;L#-&EAlu^%y$v@N#%R8KGsXJY$jnQ=5NcMY^VcO*w`;ST^e_!(l zawM|M?3?WAD?aEB@T%6BO4JmG|L;TKYGnYn9@cun-FgyVegp!nfoYE=P5MC|5b6<< zlhY>-PMx1-0*1%Z5)HH{bL|iVz_tv0s)Bk`rVhR3=l3|bnndv4x)(|cAxUXUauZDK zNG6qTOl-3O;y%E<*rqsCsKl<{ZBtOvCg0878=!QouhW2Uz$%P|8=s>DVD}$Wa8-`i zbsEGZiK(!I4@E;(VrfkIW8<(k0@c>(AP$j8A!j7cuS(9Zp$^a z0Aktmn-Xx=Gj+n_Af#7C^uQ3cp;?@ctx_nXQ87J7MV}?;qy27K9WWShUkN0*TEX(| z2VHH$>9D}|-d;ER55CzlIw`WZ5I4F@{PIc#!+cKVM_m+pj4Q#K- zPv2QF%VuN@OQvSKnPYpUJ?9iVDc6RngldSAwU*F49vdsyA1XnE|Gwz2U!0Nd+&&}gG+`JVL&0dwCJSRMeM3$!QP zM(f^y*aDoC13(dhKnZY>1(lVRtgiB@`R))al@HHB8=pd#Z!z6MYCR8)@WHl|u(Rq* z5)PW)6D1;|YWga`>G#7m`*q@>&g&pmA(xp#AL#B(kW%kU6vlxKSSGPvLSBGFkDQU( z)8&HZ6h47wHg3Zh{Rw`)IaJ8(b(2=f@`t);=3k33yFk|HHk?)_LlNB=F@a4<9%iDb z^l|u4cG>B>jXx|h4Xr9_;oP#8b4Gb`O+Zxjbj3VhKXvcz?FD1vM=)RcK3@86R`!vt zfvB-8kiQCUZlJGuVd$|g_3w6u^;aA_33>2dt;ArM&QjYV&^ zC5nMQnaYLjp)oWpS6-FcB{ANB3PFZljZx4%h+|V74~^zy73n@oIr7T!GotA~e;R$Q zhG{=uvCKwXd{e2KJ1tLkFd4WkrVIJMLZMMzQPKV+{Cf%BX%LX@zFz~|B0E|t{Qp{j z{r!Jg!DOq@rUMgudu%}1fI;->R16vc_r0|z-B}?We2AJ{enA1B`w3050#an3It;N1 zVo+}7Co#Jsn3A8h>j6RNfsoz^RQT+OkJNw9mU$}_)So-r;y1X1o~lT~4#AwCq|m!^ zUBX)rZ09-S-mlQHQe$N8NR0ch1-8g6zM+u~pVJRT+cV7LA>dB4wJHjy4O`7*L80l| zWFnBE`dGD(=!fwBs`-iuH{DO+@w%c6(VP7)zv2nxiuY+`CD#J@ZSw#32B_`> zVO87xd>ewkdV-b3CABH z&YDtM9V#t)T3M2WXJv@?(nvb}YTUy(jL~bZ;w<5CTB$$AlKE7-B{A63% zH^sGijTDeL+~rU7y6*|Sub3cN-{B=}{eG@?sQmG!-0868(h)SifNTeZVh)o^smcf9 z6eNKg4x{gNMT*$~Wg=#`eyIt_6sHn$tt4mzn0(p=~CsvU&4NyV_|N#*myD z?3D9qP8W?{u6Vo}MnpgCtC81M?5qcXVrQw1k>=h=@<*-K)ARD|(zdcDN04!Vyw531+q9W4(0cyp`J0apA zQ3l*>ps4`NF$@mqGQeaz4Ox8N#ST_!r&KHMm_TeEcy+WBG=#$E=E4s!jTJxlBcZM6 zBo9U@|1d^ijg$cl8}P1N^5T}&%$?WuE`wSSfraeIz8uRKhJ@P~(0VO>!#DGSBpQ6(rfV-(%Eai&kX8yF!s%yKxF0iBDs_Ay)GA=l3QnaU+)`kMhwGH~6Iuk~iC{NxhUgayyTmc50BSfKO+_?&bk5GFnBl)Hht z4^rF)^U)IA1<8P5rit3m&^=d-HWznIpmZ77{tCPgyyjVm?K6%JQc#eYi{q7a_%_L2hCe7{e1pg`_4g zUui(&{tOK{fC8%+@7F%bEhs->(-5;qL@bdP2fpE3A)&Ij91IENW&c%yaAXmyR=SX? zTfFK@Tzgh=>EAy)DyH#da`U6HkOykJU)J6ySknmdBqz|Nsn~mGAymYix=Hu4tD$Rv zX}khjSm33RrSjn$QX@39hMC8BOjE8d&qas{m!NBAhVamR#Pe#EpsnA)b+Lx!`~w3n z6;8;})mo$-KL=&z2&a&c!DV0w$oT`JC5R0?B?1sC<*F4m)`|;f0{!&|txR@|neo7n zZX5-to0+qnB*S~_X6}O^0+}j$#+hoM3T)2AIBB-EWoyUD<$>d*hnbQgfYO?_BdT0~NViXW(33{lt-okgyo>V^M`9j~I(G1BeMd?#@n#Nj0z~iCNs;v(4oy{696wgel zXSHdT*cA?B{9kJla&h4Tq;~Vm4?I98M@QMK+7~7w%+bk35S4j(I`=Jf=gjS-zBt|Yz#s=Ut6Nga^^W! zd;am^3fp5^(k)!c8^;n--_SMmE=4Zx7Z^Gam()uZLaj zNrID;U&jp47?vl3oi1;$lh`K-xoreqc^uf`0-82ay4a4j<0u8X-HYqGnwphHTLW|i zfImzTHd+9se1?I)#>3u$1>K1)kRT?lNz#FOPxq$Z05x$E!ZMF<;% z@~pmu$-79GrrXNHAV4uT#-)f#3sAQIZX@kC+H5KXM63lvi2vj%roIjZzMl?ImM0wO zP~Kn?7LCC|>hCs1nQvsNB}kCWXGBsn$-|$<6Io3NcXfHe#~p|f$Tm2(8dhcg=B5N- z=>Pk`av*34U|Ls$ZFlk+ZmXz152u;23UmOeT5wwn2g+Si0Ld|m+o%EYZp51Lyvrhz zcwSuYgvFC^YQOOu5e7-l%q$;%g)pT0Lz}aAfk9~!=-($-d4#Vdv{5d%?*0pEjVaL&2F5;rjj{5TL^`NE&m?`x)s3m21h+?$LlINOS? zVm^4!y?M?^u#pyqEt9>87&GYX+i8d^$5j!#OMrg1?zrx#vruC7=e!S9h1>b*GdUmz zU!KyZPnj5I)$;;;42T4g$=tr)9=5ePS5YJm9#3RG;DXFmQqp2O8!;eB1=;}$T(qe& zOnulBwF5*(-#UyJ=O{r^(#|9!=q&Sf%(6y%*>&xZr~NqB`4PsaMeBLhH?eK*w=R0q zoMYAA0|*hB1F$x74TUtrR5!DDVL807AjZnezXJ~uP1*E9Sal_Dklm;RMivo*bprh=Gtm^BL;&ffJ@ zV9YQ3Dfc^KNK}RN46j9+!EuN*b378n=#%u!(bkf4s zqNi@aDt3g5HXI!3F-jBL8^Rra}U=OsRc)&nHqw{|K9B?3^q*I?$x zp-{@T8jZNv8`SYnywamPgP7beNR6QBH`zPVh(KN>4}RTWJQA*)Dy)F@Y}DiCz>hHlTqmBNvN3 zPxZ=*C3i_zrd*~a9-p-jUu_)|zRkkqL*U5t&$i-_3^7$mCZ$-tjMm@p;pChoeIOg2 zQNYeI0J|CRmrpdhF*}$V?s3u|Y|RJ*v*7?lVszC>Aa^dK4||iR1$L_RS^M_umZ9Iw zQi0+3B&&m2u$fDt?#&!WGv$wULEBN5+y&jFj$~zH1S;Z$P><>xF^)ye=!MkdQS;p6|U+ZY{E`q(V>1klfw6miB9LU;^ zZto!mX|wDYK(l86qkayCWPnd=m6Ph4e6bnQe#N=2M>9U!4jE8>0Tx+uUK77+#YFP| z_3M{4zh67guW?zNuY1`ttBg)%`4&L;xN__U3ZM3Wr@7a%hGQM^SUIG0+5^7kixX3! zEZmgEsU_YY&8zd6_EyBbldp8qBPcEHQ+JuRtX=m!uK;Dza}3 zVlXs_H~0jMv%(Yn^Q9Z{Eq4jAt7y?GIi^Tl(1pzDPb)N*p<7$S3kdxnPosVpZ}x$g z>t&B;{Gl`))GXS<`aKa+L?4~`*7xEnGemCZNkT9=P8$Z91nSP#)WsWk+Np1V-XpX` z0?Y{xB>DOIU`W_oYDq1~@CXII_CROpK=PEnxO9Kus@2>j7&~y3Lw^6=&gV3>c~j_c z1(d*?ni_}>_M8JrhWCXI=u@8faTSz5AUnPU8fI=}Atfy>TQC9C)&_AXwaI|GOyh^3 zWsYc_^mx$3%Q>8^%K|n0NrG~OfeWbdcm4i+sr1Y1&lw#EHtaX^N7vGcapsfK(mpvl z#sGn}HIrYvsUby7Y4pxC!~kek!OO|qYb)vfpLU3nSvE)gE)tE$_Qw#cs-XF=r%v8G z6=tykL~Zu0O+w|7oK$>84QDo}4KUS;N(iNABGs<%rz{OF<~E7A9>XqxKof1b}8C~y-M0*C#M`@N1$ zAh2U+XJ_W-)*O#Do^|sAe+bZQP-nw`S$u;)R+BaKbL#7p^;+G*?fM9iHxYd_LXD(h z;Fla2`1r8>#@_#?F(Cc^{$8!n@yV>4%#q5iKC+VZL#fq5r0P$Zs2--fhZC#46NuY} zzR8c-A9FTL=SN(-+Lg09H7gUDry8w#8WRnWLhu1qb~m}cRzGITaeQzB?D3y>8e;0r z%CsxGU4GxQgxxS_F*9$Ta4WTbb*AHpO3di8wQOF$tf7zk_`b?Gx_HAM!*GY7FCmdd zOCrIe7kY4hb2luyC%q(74M#vNqs7dN>h*_x74x53uv+QT4Iq9Rr8mx<L)12|>@AOJ=c(%CZL;sX>eFkYGOooWFUIY=tw&19@s znq&u9Ns1W@5UdDh16|3)gfLmWp}#*d4ds72n2?G+fU+msf{}q0_)stf0rT8PfYvl` zZe$b)_Vy!i9U}iXNC54P^yP0rRsi&$$w?r{;Naw3y}XfWV2JGn#4uIb-e13D2eZNF zg&6_rlv)cXCNgG>H~?hm>FJa78_UblXWibCkTBiRgW+pcxg88((m)soF606mLU8HA zeL?WZgaFu}5C{aUTF`taX9XOm$i|^G0{7lEr~D)UuD%cXILQtoa)6Y|lr_Q# zA3Qj4v#TE{fae_%5n=Y(qpWk=^fR@#hOS3} z(Au99OxmAZDOqtB+`vNg`MKq4m#4@$6HD$;{3V z22wb502wzvpVp4i^c%3HJ)5|F)2SFd$FKI+j4mL7dL> z^Mm{Q^x1d&7Vw$Ib}^txdL?-E9%Q=N!8k#D-aKI09(m8UUPX0Wq51>LUX{bLpx{Y- z9K8<{4JFu3MBI*lA1sbWy7!?KNXyG_5BX>Kq0*;cLg+EyKSeerWX=m-BXz>+=X`t9 ze9C)p89yr1GHJCHb$9Z|uC>)*m7DnI=d>3n%=k|&aV?()hCD)-gnBiddWPNIH@Q`+ zLR&Z_g{R6FI*t;<^IYOawR{e&%74tzit&S+UP(tshxIQm`hZxT{kY`L<&6_>CIBu$ zm0D<(g65~__^3dc2JDO#M|?3Q;7m~9gDpmiiJXK6>1{56LO~c~gm@k$c|rHe&jvg2 z!UrGevU6^o!Y7V6B;fl|cgJtk<$lA~K1?DC+rV8nff8~t#|UY!7i0Zz@pKM#Z8wU zmvCkiY#H#~SdH;u{Heo0@x?4xi(UT)`6+ldlz9Q;N~%c=*cosCqyK{ggtEI9Gf0t| z{a2Gd`!~8sW$((>m9H%=ykMh%qVo(LJl5eFke5#B#Y2Pvh%phOL5GL-phX1&hpBsP zV9Lc!)MH$&%y;c87=qb-4Rk`J zA6cFWYhc3Z;gU>F-D13{Ot9=otGm$yg%mR-4Gjt2OmjoS&p>UVriKHY#EYTY38}f% z;GYJe4QHO*am&F$@YaAC7;|*040!LCoUg>uF$wz zb7!e`xpXcY7!wt|VF@Wv=I4eqKCSkAqH*!76}N*FD+1ssAh5H56QG&h*D7*yBVX=S z_wf>TqFfUs&G9$TmdvctWqU6xC97fYlg8;=M3(cTnYR@(J@9bKX18`5zOrT&(+|hK zQ8oa!lvBdEkFwrk=SuqKJEvfWiL#;)5E8;~w}ceHu&LGkAh)n^v&<|J5%vT~M!`Zi zwL*lmQp)^t0f0z60a~T8EpV3|pH~KCT4ejB0B+DpFOz@#?ymKBLuR*TU3!D|jGM#O zIG|Yk@9zC^J~JVp$b1jbOE+DY5sr_Y-r`BqjzDSb zhhV_{?dw_a{sCo&{JwmZE5pxkrcCP>gu|xi$8ahWiIanm4B6{5~nG zcKSzKmxwRp^tM$&8*?xXy%L?n)1#Oqcw>o))1rzNn z#*2J!^`PsVBzxDQt}AaI!omp76w$R=^_>&G9X~E}MHA3uV{kn-4hV+QHo4)b(b!I& zru@Xh)8J(7Jvfu#1=&K;DEWRtIRh`WL~j#@^uEy0x}*>Nlb8GXiPg-EKRL1h`yIvZ z)e%3AnH8$35%>ea)SM#W2{36!CbhJ)Gh1r&>drk1!35Vd$Q}rZHUQlbsKtK+JVAO4 zDfEnadU_hz#Klm5(G{INSN3~RIZH>liIrmoiEhgO`zw9tyezV0p4Q~=%e;n$?;aS2 zQ>`*IjlU$BMi;cGTh@YHf!D8<)jy9t0z|yJq$T zk$&uCWzkk8hTI!1YHVzkRp`8OQKFxaZ(2oSmw;z+FEiws%ft@2)I-XM?+^0ahCr#- z{J%iAxAD)BK|NVzGs{R6dmIznOF&-PeoYG`D+N<@phf~dGP~2IGNDO8#tr`NCSAr5 zOwhtcTCxuTqJiE_JX!OVs)hzQ!Yt~7KLW`h1kxu*#kJ_vx(HB`HRtBS`(f`z=GM3U z;YVLNY(w-N5Al)SeVI?y`)?%!h#VCFx3|BOmVUQJ$<2w{Z4U|R+|A+*ob)}mui!SpULVC32!LT z|8W6I+zdVK=o<-FUj2O&bghf~Bid@SCR zsR1|O+!U>E_~KcyDn-o0ZAH1%?ChL*xk0&)IWM%vk)C`h)IIBIn|tCdfPq!c!!9O* z8Vdd}#|2!ud0DN+2JBG(qA1g}-qfJ?miSYfE@mBUc~)o5fAG!RV1=Bi-4~Q-f{88*|880p=@85bPHjc_d(I7xyLzz6e^W}<# zA=Dox*T#d~5VU)lcy_J$n%!)oX*k~!^JC8r#jYcn__+QvM;+XldDr{gZ{F7ari=tJ4xT5V^+xPwvr1iPjGz&wb z?r6`Jn=$HkwCDW!O4_Po?hi#GTg+|6@0{rcw5eirCaS;ocqRP&TLd<`3sU>YLR>b` z>sV6LJ%Wu`&4}k-yCfh zuhE|+8t3O30+uSZM@GPXS8qs{8uu{$4BHOci{%9w-elTZ9v8>+`dO8=%kR8TkjbSv zEev6tkV3K$VUMZq(N~1kw$E-l29Gl6%cemG=EZ zIt^WTe)nK&C-V-g_ZBCEy7=>d1n?S+_R&}sBLA&}ny|aRLfoJg=6kS?huOUz{*hka zcn>kW(K;@n`vy~w*?eh2=;X}vX1L*ZoiLw-viuUiw!()>yeG0@yUph+nyI|D>57cu zrLy)`I!fEFUYDzV-SdJ1pzvB$FuEmrK3|%ev>Ssk*(8p=Z!58TGRhW z5|wnoyf$xhR-tIp?DSBB0NDEgPr zccICRULG%;N+lmJqZm%_qrwZL6NB+>ZklNz>aIEjTNsrU(Ym{q*w zdhtsC+UmnBDL7cIo^0@xIAD08ImBL77&q zxn}2P?S|uO4+rdJAMvGEDU@9>}{<#XZQq}GAXbI zDFCQ*3y@}ncS?7k2r8OGE#8H4MIkPLv6)Pm)Q%b*SFG=Re=P;h<$ zPW8c80aqem;?+_CLczq`-0JsRvme2I{TF_TZOI&e?3RryuBMMdsA-tHle*W0(`I8$ zE@^6Tk?S45QZ0MDyeupD%inRqju!078-riv8+*P5IXTx>w z8=SqcAlJqsz-GmNx}ij_$BMwjWR(Lwi3OM4e$v#QU%cYis>17_U8jXS-gL8Se4l&%o}rD#$pQnVHGs zL24@1_xn(S)>?15L2BjjS`|azg=R#keFWS zpFPp~WlLh-5@pFR!|1SbxoL&otN6Lh?#*6oeAIl1pWs4yOIO5QHE&#%rF2nWGz0$F z)D~NGP@r4x7{1x(%2J2?h6*F$8tTRYhirk%wtzU03x1QZvp|b4)!kH6gvG`5Jc(|Fa$=p3CNFwxW;cUS*#o5fxums_aaDDU;aGt)7El;{^2i z3d59Gj=XUq(`jl33Xaruo@I@{L%ttBaf8`G>L}OV*o?8BKugN=S_&_MTz%AaZZz97 zCXV!S#5{K^DW{uJx(6mEvp*!W{A)R7hbMjxN>NBWlFZYjm7AWP3h!PU5`Jv$JRcN% z$iGpdgUe8|-m{PwUE6!~irL{l7i1ad;L5)_M~^Y-=*Ay&Y|*W&ALDtb7AI?_pe}~I zH1FKjGcXXMRL}?JQ3;Y1ifYLK7}?$3E&Ri$oze?jn^^j^wp5=0uwiOuCU%Dc%tGmP zo2vFf0cp=qJoatP0D#ppDfBdZ6eZA-8g_z6s|h8_5C zx|_#-6ZzjVWIr=XI9!M}D~#xM>3QQ7DTk||JtzDbF0h zVd6V(;H{4)&Q=Unu2Hk7F(*xO-`4p+z9v%$!~3fW8PJhke+})9>p0&GYIk?&Wo08j zLPdyP??6<^KYQl3PtmN&G0ui3;B5t&~#1B&bpkAaK-W5$C!Pea06kB zDa-->@L;Sb1-9O5007|y#!?`>JBNA*`Gx7Y=Pe-RxxnligZG1+|@yG=7z)hzygSn=VvJ6}stpzE0X zFITEA&aicXnv4FVBpwx=P5A)AoMr>@%Mr54%jfQlL73)NZ>X6^Co)UQW{qEN+b{$+ z@Qn+|<@+=k6}Dn_(eIRUw^p@a0)R2WO4_Lw0C>eEC20xbwX1w3(8&J~L%Gfzzw7lD z_h#KS0=2%1Kn|84Apan5hM&fGK1g*mArtgXPs5B5&;*zUe_^2|h&ch{H#zuJWa6hu zc3@EuZU+Mh5`e9S!CN!86^N9i5Eje^@(!oIq5rwSIW_BoZ8bVq4MRt+A@>w^)CH?b@!dx@FT^v#vsM)q@KM^v*G89BdW*_pW_trEeQ7X!U3*rK z1d|TNG*$5ZlG`R>^!4jC?f*mMrhyCB?X&Z(0CNI6E3gqX z`cNfSzm6KK@(qrj#yJKd%XP|PeC25&5zvj>&G@!Pl_?M01V8%t+|=2aG_L)T#Il)^ z1h&IqI%&u(2}&@4RZB}!MVOlBP#3r0X;_k0{CyIcGH>hf3bOYOOPu!T3JgJ>HT|2#xH*|Lfg<9Z6`g4A8%jQ^WIwSG{nphc2O63#lSi)i8W6} zvGG+NYfx67MdjB?JpX94t4QAw+X5) z2BwW9u*rtlu}v3RO|r9uyM8+LRx0$xP%)dMc_Z=uOKu4;0$XA(%Fsu`FVx~)$+MHs5&MR}lM$^UJT|yn-`LNQ&My5n_cDB0R;patu)?d<0l)W6bHxs z8+uf_%%4iY4^dXa&(WF|!s{=WusZkt6}I%K7(iRGKDx9Xk7UW5Y`?%Sl;Tg!>Dwa2 zgBf)X-tou^Iv;Us97bFHqneu=rX6c)BOG^`+pF(;x- zG!v!i5?4;L)Sq!AGjg9}*MG0P|Jb{|`}3pF(GMR?ojH2#Hd(}%j9AZLX}^-N-=$25 zP@$h`1E&ud^MZjvKf4^pJ6=(C0<+#o93U>Wnymywr6@e6EY9BTu~cBgXba04A7{r1 z4+c=vFUz-&K-6Cfy%uPi%cJosD#Ow%#RTf!_264=s-aJBr2Qtx5zA?4NHGkZXa5ih zrEYS?2^Ag+c6Kbfm;j!``H}vEVMJv00Aq&CC3h-oiWGDtVVuZa*6*M;+?B0x- z?j*%De%%Q_CSvd1;%ni6SNA$ypxQNOH~4eK<;>Siwr(bq}oHKY||&;0Z#D@E&S z^OwkYQQ^bV>`ZKc{mR`0H~}J_@|e8bvH2hpU@d7-3I&0&k(R#_+KVh~@R}~yHq?PMqS9*sbOs9^eTbI9z`zF@#>$Th7;DxJ6Pwp8-!a76L_fkxy zTVIFM>aT@{ag^mFYg{KYH(BYo)XNx43#i?4Y}w%UZCs zK~gfHM*OYKzN0_g8q1E4f4aE12(mbEne_Qh$Z5d*9zZpSdPA$sCl^$hO92mlDU5pH z0xykDb>|sRE&o|=Hb4|WXoMC=J?=}&@1l{*`{xZY2>_k|oy-`(tCOQjxsQl{D6Xl# zuL^x{F^?Pu}-$)jpU3B(6pgFYP*~<%^ zJ_Vxf+6cth)s z=Y>>_pMWUqTdd|kNfWdGB9qw{eFSKrHbXbmO@L%y;ati0#NhBB`iZT^EO|tAN@s7ph!PS}DtEVW(QLk%b1&xct>swT{7DURru zdG)iYCcQ0z$21W2;!mulF@-uu@7sJSq1`NY3o35h#acg}j~wLe^=?UK@&_i5BnT=r zqmph*7ohNFUX3+^G^_8tMBHuuP$9QxFoTLW0FL5`sqMDmHVZT|YGflY5hGImqJ=38g;q|d!&*Dq( zZxz>n84F=p7SoiU91oquwVJ?bH6cY-(iUp(hg4r5nh`Z5h7qL#uBJWuh4$MU)k=L4 z)8+JhK6#U^Q)VfqS~D%6PKsO0%=ERp^M)cNGJP*Arma};79M7zYP02uHMAU>v1$4) zwY-z>J#BsKviaKY=FzyvGMNzfm#Pm2sqeGoPxpr_tD33<`P3@~ncj`-GPc>(mb9$W z`qnOt7j>4fE*=%F6pmr))XW6!+=|Zw!AgI3W(_QyRM;Bn^;vgRtRi#2 zd-*-ZrO)40DRtuF%x(UIlV8aQ_Q1=BKXB3IC58RP;C1q?qRM#Fgd&6a9y5j!-@_p~ z#iY~U8jXYwFk8fr9+h#r#^L?%S!1EpH8cQ#S3%$$yHumH@95G5AA#L%6baqY+*}@Q zDcr9s6pCR>Ic1jbd6yWrf>7J>XRUYsLh?l};)H-PK}m#>sys5H3UCtO>dQ@g_OoO{ zH?G2Ke3ho`GsergGzxXv*iBRgZU~^0D5|KiF~BrGy_Ifs3yU_2`SMfk|GLyztp9bX z@u)%@Ci>eu`b;as>X_QzZ@X%$PQ60D{Xxlylh643^S+Vix6lAyWotyK!!AB zm7+g*kZz<@v5+lt?`kVy+X=6UP-6ogkAjRJbpcFS8g`&GX;%W3R#8ceu1bg|Vk$tU zNMLzBPfC7hI;pd9>UGg}$LX-|pD4>YY@#4E2uT-|nm5i8u2xgc_k~9eE)>=Dg6&ZGSqa3INs*1> zV_rF)Tsb{QXF5RLoWwl&E>}#5O3$XyRvG?fr?9!1__E))8*9!y%kol!h)>|N`t;B( z55v=pwXv8k&cU7=9n+hbe2cT{u~Cp3hx=o|Ji8yEblR+btFly!F=5U*B9`ouiI9-J$P&cqhlIXK86&2yMZ< zK5*qJNRXs8RAk@wzH}qL-+11V!BBUVaJ#F)D0uT?d3^?FgnOzwAbtDDa7-rdpF@Xl z%-^l{-sStj19l(Aiw|;!g*TpS&Ea0Ydse#(1Ot~DWXsM-%eM=9ze?SeFAVC~h?ehc zi2aS((j57w-MJj@PTvn7wOhqYEc_oAAk|^%2gEw{gWE^4cvUpP?6@(%Ll=Z>6hAD` z3IaF{z$dbfUlGt~0e2OYL~j=lfks}HmSgR5FhbP056_+pEf`FrWGIhxw(4>nCKYQr zQ$(H_nS6L&efpN&&o~^ncAg%L(7bW&U+}l^0t+WJv4m*ZM#_-)xKX4b-%^~vW(a{x zBVN~(e%Kf`GqMk|sa(s8TBC}xd^2oj^(C61(l2*R48M1Pv?Tw?(b3q6o<{ijEU5PQ zZ>uGCj*gi0d`xEL8ysL=4t^_9$%HMpD8J|DJHv{yb2}<4>%8XA zh8*S^kA{6OHgzoW=`-c2|v7T4)cT84JiaV^t+xyJD4D&YQ|L95l{ z9S_cb9?NhPAy@U|?2hVK-Z9IynD^H!rMJVcj{4gUb$oA1ZIJPi+Ag9x*7zRQNSvPC z$NM9PU?f%^J-&o=+&!g$FKgTRml4j&5`9ZIHy1Knic@P_enyI@o6?pA83Z%Tap; zjqt^Lhd$YrOE|TRlf#Aeus-5z{n8YIaB;8oUjt`0JI&|2vMh@!qO6be#bn5u2M`Ii z1KqC@q$y=)!JF#8D7S?*gu;%R_HG5H)x+)&8FRH_AbsXg-#0J{n0ZQPmi+aaSKG84 z`}Ff7H7FXW&+j$~Z5M02=1|gHX?w|E2@SZ-R<>5XA@3r4vzJ%O6{B?aQf)4douSLKs9@_`nCJF^uy=cDOlFWs>mEI!@Ix}5)48qV{&J&I6+@%zG2 zk@tq1ZsWFw@56m|mH(aep}uZ+vq%W@S{uBFY2HM03sszbEMwGKP_3|{vw!2t zaR3xG@EF3TYY4~G^t6Jzl}O7Gema+6=MrP!GP)JpPw)^T_EM(u=r+uf4}9YAh~boD zKicCeG%++haWxjN&HOJLg}V3uolz@tb2Ie7H z{Q5uxI)BFoGoWJRu@F==1&=MaMM9mFV~}AKOK`??PQj6xS9137g@E-G95oy(#oe95 zP4LyW4-X047i;J-_Lqn0m^ylnK-obxNA=7cntE3$*e!E&{qI?Z=_KxH#RG%APAmQc zH(R2e`8J>X=Vks=Zdl90Rc*)hb2@?`Rq5Q2iVFEG-&H5F_<=9SCa{!|eOPPwdG`Ij zSEhSy%+v5j{$W(uk_37vGno^lqCLG;$c4wKh%={Qkm|*LTi<>`~$ZdiPmC6r;E3UGOA+Y=Bn5HlrKPT zZB%fLc#}B>T`Jdc4;_sA_c0$mG&@uU<5g-8Gaxhndl7`-#eM-?iC7XLc@YsLFsoyt z)2N`I-crO4#i3JI)YBWeyIdXv-g+OYRlP6al+*Dh>@s}_q}0@apTloK0(zifXfWct<=fNC z7ml|h0fusENyKE!BXqCszx6j}^tksrH8!e0b=ce4)>D--=NnCRfUAv)bel)|d@z^2P{Ov-i=8WXpFW}@fCU{4| zRsMhm?!vW3Ey2;wRFc0hE;~kA9PUZ}wA6&gQS> zi@SGYt3ABMHP8_XzeiH|mzVwUIsYWP=`w}VY(sl$-t^L^?do^OaTjUldB06_@FnMu~UeBG3CUMyjT&2;;Bd=xD6VWrV?ktrs$hOwL>HNB)T>L<& zHzJUt@FN1I986lKE7PnFI^Ua(qn^nCr{0G|AoBIp%y$(#hCR=;FDBW?-F}eW1Ytgb)i^E$bq+kra>dO= zLk`T0FezHIygycH1v(babP$>-Hw>X*)Dxf+*8a+|uVj+|&RXRb3#~*^VJj}GcvXC= z*mq`I4YOV@Tb$S4K7~_H_PLnuAdMLN5=k_XGqbSaNd^AVzdwIcn2RJH1~WN7Gv(T&E~Y9KJD-|;tEb1SHLM0oa=sFog{P<{xEoXztVB+@o>g;d--rt zM6arFywF!Pf1^~fdJa5EPPV&Wy?#qX4`Y0>>>q_#l~Q=6wZ`F|`*?AVbGkhTjRxIi zo9eg*K71Ni3jd}=Uey+nvx!Gy$3QQJo8eiNF0kbCZJdmyvW|{8^f7+1fB=K}k*%yn zhZ^|#cqa80HT!Wa5? zFLu|<=+KT%E-y~^RuX%EHchL0@!v2D!~I}RaK|D&_~_rj81V*DbrMK9j?`kP)pR~M zD02wa|q+d;xR&uoeN<&F&lgE*%(l3Tv^+cY{$o8rDb=YifJQD^x8sY}<% z)S;Z4?vOtgeI9xqe^0%bUwFT6-lw`rDeU}Pq9Tg(D(8DDn-&U=uRc=?*}$bMvvsby z#J4_PuCyzOHjk#n+usab4J@Z#^sufv(;y2l!#q!8v%ml|9$#P|W6GK~agSjB_Y?a> zAHW*fKy*x`Y^$*rF36Pys|!d--gN<>7lcS5#%mT9R3I(2uc5;Vf*+;?on}>GK{I)B zQ@0~m{XwQ+Hz=1HWKS{?37K4%LLEw?3L6Xd>Ma4a^24<=F-+40#xgH%+))%SZ{ zb;vhBO$%Jy{zj7!Qgp|=zmj?;Z;u;hWQY%^;TX>rbW6a?f@#wB07mb^+0Zc2T!%x~ zk^R;h(d`Huk))Rvx5HyKjAy?~9P|ddv~Xcu_hDw}h7KaK_P$3FmjKxO%wBnZrKKW@*g({(E{KTB?aj zZ<|4s<*iIp<>1a zMHDW%SsR>P+t0C>Ix9;_oa1tBo|&;Et`HI|F3dgC|ElPGnT07-+)rRNn*xiOPBT(ze^ z$7`;wExhNi|Gk3GJIK4h@$SDYYx7m+?zTx%q(T5;YN`N&K(37W1oLcf7B|i$N3Fn@@o(T7Br;LK&P*z+nKAgQMwrwO(q-emawD?ua zS2q|CkEjAYZMEqj*8NWQeZ*0~!_Dnp$330URp+a#i^h%-MC6^J7Va|eL6Np{8*+((A;6;Z~RWc&~mUjvpJ%a+|> zFdKO%B9HZ0%jJxvidzuW1!oeGAIX{>!7P%`W8ZM=!ftJeFh<;SyYqxS3A@&2Jv!PlR0xWH!K((v(;JO!F3-?&(3%XMntF;7wpUznq3+r_ zZOPuX7AM&C_wQUh&;L@?Ie(m_M<|52M_(@ar5*;-pO@Dg5vHCgkWNt%;nu)`@XvC0uH;B2I&icUx0UBUzaRTv^FeQ8oJ!0|U z%K~db#*V!)_{FS!m*<|+{ZYpi?!&ReeSEf1ZN!fwH!RJnXv%t0B1A_kS;!55Cmn|$ z$>11^XH^zhw$Voo$6`FjlrsTM1(dROWg`X!XG^|aLF#eT83Sbx?|HCkj@gq>U z1<|?q<@NRO1@?yptkr~h7zu5)T+$hQeHS#PozwbagWI*u;r6H;D6 zlX*rYkO)xK%T2`6YmLq9VVpYp0Wkut%NdFoD+8d$03crr;q2thlHlQ8*O&3dX*t@h zLMX3m-*^s`TbR_>Js~U0KjCfq>;5lAD8k74K6Y>TJnXZ-y3pW89nX=!ayeMo15P4u z&xwb>xSZt4hty_$6>>DkQr5ll)n})an=N6q^_iCptuO?06#%oG9CkKRQ4CKO$)BuN zW^WBmYrMt??{_HmO{%CMW`(m}TT^cD>`*1Y&aj!rh9e*SfU`Of$&b5Q#EuBvFm`ZX zKYoYw{J`dLCGPN!HMKEJ@Z!x{=DQ-E`;`|!p917P7d)ry>tt!rR+Tn%^eLZAQDqe6 z&9sVuQ&SP}VT2tvUMs=?puy-HvwjM>c$=HfuI zpa_JJ(w*e!gOjiHX3bz-Uo7+9HhZfi?I;BZtNq;hsGw~rkLptFo}V0TxHVBcT0NWH zY3QiUhdh4wK2!g$U>;0#b7@D2lqd)e3P-^HV3AS3LpEN>Y&ncjI+zr z{Yi!K`FWO{n@~*KrCS2um_m{A*@gQgY*LIiEh#a|RC+C=J|y`-ZLY+= zMpK-qCvRI?$xWr<7ieHE#}X3fMVFc!=dQ}m zzha)ER~?o( zd}^iYPJr!=h-CYaxB@IT>89!8Wtmjf?ljT5dXvnPKj%{8$zde(KNN!(VU zN9VWrY%i`N$LynAw$y<&<%?>zK_|D)19zYvD)OQjoR?1k=2k9%vFARm<=_W#-$c+ZN;CXw^y?hB>?>LmMX;ZGjiMalGR zX=s%l4H0)Y7xLChe%DjL9pZY>Y0@RLypp^RrY3j9MUlDWZaJO9aRC$ zasFnLF?ak6cBH6tAehwyOLll_02~h(U~#TRX=|KZ&4EECU|)FwOQgAu03l$RAWHTr zAmBlfVKPdmNg#a$VN3t-7u%RDbz+Yvepn}9hkGLYT0veZ56BD!82A0XKd?a;i;p+L zPh6Oc>1#b|A6D=`c>_A^`)WQJpfMBlgIq2Lbh!Q;Z?KckmVoZiRkJ1!9NKqD%eT(vx047 z;cSi4{XTKfkk7if(j~^E3~$|}$n&l_l^JhoNW)%7?6*?$%w1rhKc~HCSXS1d)Pwuc z;iB#0EiYn{0R+BgM=BYDP-s_E-KA8~2wOa_u;GWqe4JM!)1>@aJ64qcVK`fyzZVxd zN|v=!dV$fL=LQ$kF<-~!3vzjLYVFT0E$>%<@XzZWMms64OIg%tHY7sATC8xM_@1mw z@PkV$Q8336#@XS2&(nTYP+7G*G;J~@>I5&Tc)6k)mnffZVSRxT7%!Wk)b^_cW!0I6 zyzhC)&l+3RIFgQGo+NddPMAOCtnSROLJeLXr#6y`vUc`0s2$tfvV?&G9!LFoULkst zq@We{NdF0}V^mM)9`%OPVRd_9xj@aeW^3B+%SPiVjLYCty$mVWzt>~;>#h9P`bYuy zjmxWHUiI8z*I9N1>&{lrUPx@2A5yuhe_%{Sah6ki65={tY6PTm-ER@L-CwYnhg&4_ z*QIBYtT!EB`e&zaK3BqF_PIP}dbsM~He^~+xUcN^yP%FIjQvs@QX;tY=bwuIwRZH_ zZRK*2FlBAK zT8%=Oe}TCZ$l#o>vwAaV3dCS=P@&rZ;swVqc_eb1uNicp`v*3ms6rqE9++E!p%UvN z@T1#8*kzr9JawiYXIQAX^xvDC;ce}ge>;N_#z3}nRe=rxH!xj)_y%}52=TU>h8|!> z(S5kkl@mE``rRt4s$%$RLkwu;kX=Y4YklhwCU7DrLl^N;8v8=EQn=KMektX%Kf z*!4)$iqrBt&*D+F+K(n4%RPEMaC_pl!^^5UyK1-R<;jfNdVCCzMQ88qQlXwZW?}qH zJ*n}g%Vp^GO5bbP@|YEr+|7fr`jBM5F+Scv;O9{)+Qu6lxZsyu9H^j$R z_WX|so^yE!bB~4&)#(tJ@)iub3N-V(1K^ib?ZlG+wHZ+*hkYN6CLe$IL#mJ zm=X{W)`n{E#-wG0I!xc7NKV=AG+a*|097%yAIN=PalG&MCd z84uPX&D1$<2)pKI!aEj{rF)sfXA2|OwBvW&byJ01t~`6rexr{FTP7#@W{IN@9hO$A zt#(Jssc5?8?K1aO_PWor63<;29n@%+z1)cK zsXv-b>-VoRTo?XzTV(;~4OMR5F-kUq%Dlf%H1?~k!?M~%Ta{mf)Spo)XJXk8T9+4M)0*qGY&j@M2O z5@=s_64mdyxNsb8``iq?&D=J(vO`hAm251{(cfL6tKa}k#ihq^7{;$Gw@#f~y%vnM(6K|BLv(V9p49DXM{Nz&x9&71Gmjx{% zGFlVS-P>=3wWOjJX=nbqMum}i2LFh9L&=y`*>L{g&a2Of$H#jHI&-u2*36XR&-*uS zZLKf$5V?4&$`J8&lxGa>DkS?fbN{(^1*T)_BZgfW4~>*jX-#(|M09CP({A`H4_ixD zuW5hGvrw*BsdwAfm%i!#1UR8%AN5=7ev+zk+Z>vGj}K8K=}vVwsW`fghAsC%Pt>w2gY{Q1FSLr{3$g#n^@wC&LGVk z5q1rAbvr6pSpkjD#l3!U3lQ{xG(`z=&BLp|!OOq|M3r0G+R7ak+PcjXUs>FE`>RD8 zmP)8Zu8F;};7q9^OXQ_!!?_%~`kZnGp;`df1#z)fE+CY^`}A`I(W-+G$U1pDn`LNV z;2UsC`vYzee(aP-HUrj58~a9|JdylAE&v?R5+QkK5o(}?fdBzppWj`77&JpVoUjqN zdGqEco%>!%($dm9+s(KYR-=Wbk2MWspP-?gwIfGef;FAGoi@szwEG9v4rt`?fj3}O zLV~QXuNXx*CeWtGO459!K~JO5bWBP{W_KkgQdL7^bfZpNL!%X`a)CyE*8#0dWy+3q z<;9B^LTP>a8F?%r@V&64)37UMR52;1pzvo2W6-LSh=iovW6ylXbB#0I^ABrw_D6bu zaUPQJv4E8uc>IdA55J8{y>AY0BfDHLr1M#a)4bw{X-j^$)hXVLy!1lVD{atHs&u^Q zq)tIl-P*tF_PasT@0PqiYYXq+vsfIJ=Z$2DU*uf`a zuMPaE_>VWsT{SX^$8St- ztm;~ij_YgxS~nHpygTGFg3LIZnYvUh)O@mp)}wFM^U6($?{M60;`yh%L~Ne5H(s&m zFTR`5`c97eew-v_f+Ja!KlMmL!kiZ;SIskkJ74 zk`jfZifeG#>>XON5xlC)^km9N*TX_5Gx+5OGbIU(Y^Yh}x z(@Im)f+A-8l){2PD`Gh6t4WR*9J|IGSRGY;n@jc5?3QlIrgL?VsE)b1l)2@&$xthM zbB)StL6}!A&v+f!_N9vK$C+yDtMs8;?+lb)T*AhLq&SKeO}}qZGWV7J>!*cv8@uk{ z+{N`}8(|W|kr83+{@_Qk{v#Pv9bQ{6+f?l-otSW13L}lbwNxwVRKF|f7UX*=MKZ~Q zDe?!W{Ghnw8|y8Wp40Y?>g@MT1V*iv+ygV4S!Ht%HfRKQ`b_8N2?sxP6?g5s5H%Am zq6B&IlN7zK(8;x!<9v6JU&?=-%SXG4eai4%lwXIT?1PU(ZbzDE78gx^+e1g+%kqwg zG@nuCxJ+5Yey{&fsHmt^M<@r2R;EIVUmao8ydx9+=8`Pf1od2 zTC*7cya8RtF|~JDzVR{QSsw*&imJcyu6gOw97CL7)ZQ-|2`AQ21TO^@b&eG zdee=7l2TDmj~WE4K7Zy%$!CMFSXY-qm<$7C|IpTK(~5T|Lh$28Wm+IBxBx7Q#`HK@3fyn-@$oy#l^#F7%@iYF9XTkeSxs%mQflQv9vSbv4g!FW(xn;iIIxw$CtNvIVS)Q*QRGzUBU zmX$i*VzZ2|nhm+ReNWvHnai8EZYA-Wb;VKJ2jgH^W2l;No|EY(jQSu86^6hIOCV|8rAZff^}z533-Pj+f;(+;u+m_u}iC*vY!OZwc zs0v|gkVmAG+g`Vg`~H*9{~~k#ZR_Xa&o1no$`+Fnw9e&Npep( z5q|4U5h%A^Be6^2D}6o^-k03*`{V+*@@``%y74SnZPmOhl8VlDDYE+IAKGG1{HPbg zmHw?;bqbP&-zJh6?x&#~gT6S)PSltS-+wYP{}QKcH-~s4w4H}ve6nB{+6tJYJa zv|f&%B7A;*jcq)1)$-+X#lCeCpWC@vg*=8Wb`HsD(0v_!^9}Y_xgRq8|M+_iMmA); z+f0(G(|`RsGH_wdW8q0DzMI=k=^RYM(HB9he1Wp(N3wONKNQY)Ey&U+rKh9o128#% z4c0Qv9bWHwCV_`1g?}55oY={x-s4@ z@ppHjs-rVUL>7*v-9vEY>r%X1-vaT0-12{%1cr;_QHxo!taFUf8!De#iTvrX~p#7>LIKsAd>fOuEkpGs`I{ zNyVG>fx0rV-|(gJ6S@&J2O%}VG*&s%%*j@O64b2C+astS?2o8DI9|!=NflP(F^(kT zxdOovdK8523)PL;uWxTqOibnOJH~8mY;unuFG7u#^XU_wJX*--m>;wi0_Qs>>OAbd341ix2u)L8{gvNVY0>Y;d zpm=wD59vTNddWikg#0Qdwjs1gbrYAFns|6XVIyw+EZ-T_m0@!pQF&&YBi;X3! z)4xk8U}L!06{nOY5(nLh9y$(WEWg9s4h{=jkH2ipPl0@%5H4mR+Clho1l1nsRRT4O z%%CTODh+B$cFkf#s3ReHe1i80Qmw^dE}*%LIEKe2BvhF9QkeH7HUS`_UGI4hTB14d zg;G5xte&dkcRtK3-jEY9>8)910CI)u`x{$!k$?;;$~*PUa#K|#w+br4hx^OI1&slkq|yI=J-Llyn^+H z!;jwdQ%;8PHkYKxZT=F{R3P=H};@`}|ynT~xl@%uDh^j)%W4!;94jE5iQS&#yjx z@x7EuVtZfGiS;@k?ha_53UNVmLh*o(T9PRjogOb2aQ>Z_gr1Chk?x!`pUkUKMJjPU zH&W-D)2p-)ag>2E*5FDnDnHkxN@C`E*3+QJ8McN8AJZ_dOo#7gvlv(YSP?Y5^f)}6 zO57DYrMD|8>sqR)SNv{;nX|6?=KBw2(N7Y5d#ZK_<>eo5>^nVn@aQ>d8pamUP|z#N zs5^g~n_Ccr_o8llPK4mybm!ey8SezFQLLIO(Pe!UvYXqirz}nwxYw^?;|tK67}t(= z)TKw6CGA^3tMK_brWm#lnIs5{{pjo%Qusuz1+Hg(#13JIu*bX*G4WzTM^#?lTj{Na z-j(k>(|oQN!5t&`-dh=j!cO_AORh<$Suvd*N{mCXH4;C()7vvLI z@SRCw=LvYBl6;QoyLlt3gy?gOxyc@nXg*VYt`^StJ*3k{lNtpNDe_g1hEN4hAgot6 zS@QFOE)T5o%IfG)VEWHl^8|L5D?z?51;s09U7&4A09{^4)^V97NWB3#J-6MIkW4s5 zD**aOqouyB5J4U*EB6VOjO1AF8{XI3?;Xo4dAWIdps}*;62Egy5Jtx1hp<6rFU}86 zcOqYzHV58D&K(@{>jlHXou-KW62vY(vo0d#^at`PD)=UCVRRxQ$%tta=x)6hKb^x3 zQCmbn7Em=(N`H_9VP5s=({F9zlt~Xfczurls!*|k^5pkbXEvF=kdT|cAUz(Fmevc} zF+|sYIQ5Gn{TAT)b*7syDs^B}|8iftXg7e5hm-xjZ4gR4z^Z{v*fCetk#&6?^lGkx zb{#g&gG3&aHv8>g5=gKEBHt?zFqek_2?8-j@Gk0({}e&Rh)f%@r9;WYyoIh^yCx+o zTLCp=VNp?MPfxBvqi<|%?7-AiPvC9Jj=nzgKO-MNqgX*v@rB){cY}FF5>$EuFK3is zqe3Jt0$x=`U0uq@M-&VQ&3<=8uQ-bU8SwCA_99|<6{0kz3K0((Hb@yk4G9@cG`>Ux zYzfTF%!r7IT~4Y|@<{QBSQ?%#-hDOa)_6j_HeQNIlK+Z-Zbv5|Kq3Lkl~(`?3WSb0 z42>p03PZRz;K&OS(4ci06cS<#Q4__?3`4duEe?pMwqN^}WJj z<`IirEA*(KVP7jO4-eKMnAxWrQ`DJHF@h;e&xINX? zMcyhb{A}>Z4bR!*5fitji+a5;DXA}yk;0=Dy|Mc3GR7+`?V*QqOatCD!gvn{EWM^` zb-7646M42i?wCV=(=e*Wd{;MF9v*h+;H0Fa3@auPOo{X)ob3v6oHh9KmU|3acsIB> zecf8Sjj5_OoG9SKv`5PE+eVwb)@?&-#FFem?(25nLlc4G=iTj!V!sEP{GrWcX&e4i z$Ts7uvm_U09p;3TPO)K){_ zv8R}RdE%cxg?6)%VIkQUuitQJxiiPuC4FxcKe+{=A!5*Se&&P*b&oUrVAkuMi}m+wf(v!D=p1q_=}cg%7HXThIDr*^zPGgx4Swe@mu1@ zWYH=rbq9UVZL}IAhWISDJsa)gk(ns`QXY$$jcK(&Y0fWI*qFkqH!#hlB=49PynG9vG6m*)o5`#v6@{smi3J+h21q z=mu@_C-tBT%#15#vq%3ljEbrM7uwj^#>HK0(>(T;cwP*l0B7w6eb}Sfi!%ejX~=}^ zX+g##A$AOXmo?bk1CF2v$drNiIN}NR`}c2W_74Esv}9bITEL#c*BMWC0E{4GqZt+5 zL2Sz7c(ounDCp^(v1ctd0(&4V5uKRW(S5hEXQtlk2pjkv*qNxbQ5Bxsu_|D#5Mo!Mh ztYswFn8-*7u;}0CCtI+#1?=D?{ucx^k;~;~G3Djuh|1dPZ~^XCchPS@I(q+35Iq8E7rI5<-!q&-wfdOW=j?*>N-)?WbITlqwMs zIB=NXK|luB7kX%DILZ0Uu_fQ(w65(to}YRkZ8UhQf(8jaBja1BGYRnV-=O57-ax0M z6k%AuU9|f86JRJH+2;kHjeJnk2JKxq>UlLa$?NMjAW@E}c0*JH4etY%P(o&9V0?rQ zuhz8$<55%hK6!PlrmCun*kDamSm1gcY?>YaT}4VxLx7_p5-ujKI*XvuXt<4ZL^pXX z2WYV|uR>Z{1kLbe7)E1=P-{St|6S29rY2VO@nb~e`F^RXxjC=Z2n#(uJwSh9RKlv& z%JF7mC+mE9M9(13USefsg$NDH6ik@dWZ$5K6wItxt)I4QH?QDv`J)cY%buNQ*8CLn z3k#F5N)3C{C_cWuu25Qr8xkW~FLnrdtu7@+jXxe34R|m3&r|cZ-g`&K3Ul-d3SVNy zIV#{0FZFqPQK)wudV)=m`M;u#;zLo<;`G`v>WKdNdCKETJAGYW-b~eIi zib-l~!(6;X8$=%;gaxnNwep>Qtn_&^^PN{M`TYU|15YD4x~W7R6Ymv08)B|~8OeGw z{nd}Xf4qB>C;3&~6o$k~nRx|DUaGw%8@_h+7^k!}hpeJMWA>PbvJnx-b!QG*LcYtc zs?rLsGWcpo&B(|I%~^Cz3?lfyv$LbnnOa<2Tx8WX1q}z%-pR|~XoU5V&hG9tSJxOb zZPtqYZ&z`nq6WyvkPZ;e+aW0UR6`5csLdQqi^sjX(;`jR!K;&oBR|AqF({cbfo9O3Mbn`9wU)uPhbTEvCFY7JXDQVPs86ivab6R;?>1jCYpD8_(~-ir>bL zP09FLHFQOY7BYA~AfsDI;EQ1sDLeedrGbj9E`QFsfOR1wC56Gm!}G9U=M3>&gM$H} z?ty0mDaIQxJxKRhB1Z!pzM44k;3LcO4Ox znD;6sTV^vi1MX|H9ha!axxGqFc34aIP z3i1=C99Z8OMNPTV`=KTRRIupzC5OQve0<9kik)^h-$1JEg_j>CDbps_Qfntq>{s}& z=%{=njlHoSB~Oq}YK0lx#rWmA1_m43Gk7|SbsNI`um7WyCyF@EqDO!Iwf3D9|hW_|L5$|)%!yps!9lwYIqjqRw;M>p> zvH8;@ksfnC$(w;?hI46QDcsSyxxd-jtUc?~v#vEG2fD3zvRJ1&mGMs96K!o)FV`*O z;r!b@DUI8OI5L9lmG|y(octy18y__;lj3%s;3k>zAeA<0i%w15eM`^ymutb#sJRnf z=0}(|a@~<6uO3QE3!fIWx8*@dXM~GO%V0{FFSK5p;{GIjMUT{-iPP(=6%VJU1zB@ zKCDUnD^xqGE0@h~8X-*;UH%L60>uM7aJ0#AQo zVWE)2JO(&-%}(gVOJ9Cw^6dPsfZ%4Mo~Y8X zs%gthYYE*%AQAt-vz`XA@$;QXa&g}WNKfDaPVq>M0HyC!DpFFyn{N?`K4_p2 zWGFh?A*meZ6eZlgjTt|F=m1bm)PBQRWGzg_BvSNx79ZZH{Cv!J&2RKeEug*myYVvt zCR;lO7yAat!Un5aepKv;*KBKXF$Wwk)qfLP2Ngh|hF%_W9oS{vuGf#$EYe60eAhm+ zPgAVP4u^yrjoR6)-70rk(}W(W9hqUp-mLv`lrw5Ye$R>sdY1)I<2E}z+yVU*?21Y6 z$@L)nK*__kkB(M0HV})91X4P|gaC03uc}HU)pvhWX+6ICr>KN|WHx_L9acW;1EMAwY{y?}#&P{q3k zTsAl8IB%>@9QEB}drh^CZ&|pZl(RDD#K6FCcde?Hd?4tGNO;g)G8C%Y{VYrIZ((k3 zuC=EC+wATC4UW0Vq;TGfR4_^5&)r$MR{HUy@pSU2knw{1_DLkK)0j>9@!uw!FS^4^ zg@cxIiv6^MNm6r(Wi~%+-X6?&gr?nh{RH?e(=6|P{=udJCjb~wLX=y#yF$rtKQ20J zGO|LCR#W9lshL8sIOI@t_AWrW)2cQmR}?=Ff`id zZ`UIt31=K!tut?rp0qJP42u4}xX8EmMHmjAiP+I0gJy|tQ^0&DYAUz#Ni+0;njf_S zWr#d}UytPT+Vn0zS%T&IyP{YDri+&bCO_-0BWb`__e>JeErYg&Yq- zUdTs5$Fq$wkG`SX`LmY@wKu*d(A%=D9;(BzW)jC19YEV9mzBweO&oXkjb;(!{rf3l zVc1N{sU5+D%oD)uL1=9^ZrpgL;o7?BxYNm;0p>gk#>VuBx(K4~&&!KOcGK}0Wbkgj z1^CY00=I^%%|il@n<89cNx{LvL68}w!m_fmB!n+O^c*~ZR##Rsb8;jdT7Zty`kDd@ zgDA(5h3F9;hN-p;<`wg+VkQL=6B(%wS=bbUQ#Y+X<5D)7X>T%JRUwKVCWyXA|NWhH z*71W!v{-(xZFuM+SFSrT(MwI@as4$~Jw6dVo1PI<#KXX-Dzt94`2Ccn10$AG2{}=4LJeVbYYFB|Q6JWp z18*OaW*jw3h-enY0Z$4sDbGudrNsgU6wIC;f@{uhUGu>&V8}o+hQz}}M<>zAn|RNf z1myQlA#>)|)m3bnxIZ9`yz#Wl3YL?1pxu~-n6?cb@@P1+pxjt9NnC^Z!o@}5@XQyc zO9*lE30&-Vs#qkT={?fxk_xvXrec%+u4suYBpj`$qKpKLf zW8L9Va<6&o90ZuyvLkOX94evLz2+tzQFNEcurUFO^=~qSIC?cUJ|28F@=cxp{_Cmn zU24s^6CbWwlviJ$23nHH%qH;fQ;4YV@!^h*9RvvG2^ZXM(d zuX06GlJgqsXvaF@r>+P-UrhCEt|7VoKo_`C50j_zryH##LGDnw9hcDY}Zcvvv(^0TN$ z|L1;%=Z|ry*G4=rB*l(=#Jev$yRv&l3$IgzC+P)V32ItkKC+|pe^b2$;`)=OE z^oL^BR{VBkQ4#j{e*HzJ@vj>A@24zBMpJXny*abr)A6t-b-Z7BHDQq<(b{%SM4geGsSCfWkq5dl> z2@h<;Og~+{XouHtX%XVuqQVd8Vbut<24P@bM`qb%qxU7D(^#2B;baMuOIWzGDC z zsGoRxV9^nw24zXU`=nE%K;-M>#0RD}A#+?*dUyIzQD#SIM-@Qlp=f0EoLbvw6Hh?h zDAD{`f>IN@DocXleYKocfS|WYP43?9AWgo8RP`&JJB9=F&d!G1WZ)At9hL#lj652i zscGy;I4uup2YNFnhpWCbmLSja!p#hPMMXvEsTlb9Vpdixq0a>OqM#sH9J&t6t^Q4~ z+ZR+e`?n4NnU6zT4h{~sw6-FBthF`x(13J?GF&laP@ZEj8U~7b0Q@;@1gbyqXa?w$ zoYxfVzq1PS3vlWJ5N;k}%sDbb+F<13QWho*Se6EBnCD+?R1})Y!)0J%0Y40<*hU~_ z*UP9~qAJ)+YX=)a`o7eZaP591}XH;jumX87}Sk={U?teT_Fj5yRx!k zhx6?r?d$Hv?5wQI6_dq6^iVd#iGWdWo~h0=kFIhhV4?vmh?vI%Atove zl!}f{q3#-F7zePigOV6#4$wp(_=a+tYN|2UV~iRu<(0N2k_f7UsF{loq=d&EZ+ub{HvlcklYbL;Ng5rWF>7z{B z70h~?x$kqiSq*$$B?XR`=YpywBm<)oGM;b~)vYShx3;$K{c4WX*9adTxaJXH#;Bh! zU&9j8B1ARi(q9j@){edR@?IMIHC@`E!G+q&H8N#rnAyIv68%f5jlf7l?LE-DkT1_s zwb0N|I8fEblcukelPSOnhJ}r79?DiG;3KA{Qo;Rv@?Zx8nHwG*9d*q%YF)oU8F_as zMOr2I)y4UPv%kY#E{NY$c{v)$1M9A}^F8pfDg1T@7so|M*65#ea(kcnn2If|Ds`Mi z(y}vLZwuXuR72<5LCm<}EW-dK&g8vEOBF#x=)nQe1%~yJmE)7S{u-hCQt(X{5!;oX z^$qRTUKAk9Cz~_)$#M{2B>F_?zX7WIkGu8!LrKXZpz~tD1A%7_Mj7y|14#zvvC`0n zLsft2011$76`jV`^J`Gfzv2k&%%6Ny_pGSc`D(~#f zua>U_!-;sH>L3D`Fp7`3zxfT2Qt)&O4u%@@4XmxJ!Gzesfd`b29@0Jpas>U!cAc%Y zHN+7~*0MP02~&lg+(B&lDC6Sj4hh#fM|ZtNKMe$9D6WK)l9H5?`N7RZa~mEoRTY&c zxCbw6Z9#*6r-kqd(z3v{9yQ-N{|3{d_JhU?AHWvDW4?XSu?)Ndfo)JI4@DKS7K%B7NoTMxq=GGDw12Wb#!27KXIXa8eQafWXYBCQ&@BUwcQfTkohDNHYYbRn&cFQW}0de-%qpN?fXs zyU7_E{?Pu9f7DaJ4FOOO z9TNl7!{4wdw}8wVhw~o;XcYu2?|iS$y{T|yBM*v3tynoYav_O=Q*W7BHwiqO(13k2 zF53nL;^>r=P?nI81w}Lp0PHTX*^_}r2;qYOcUG%J6H*?661;3UVgkDqv!Pshr3|q& z2*PmMG5y~FO9qlPWGl1fzBDL!(5b0mKnR1f5t^i@Dk^W943VM2gh?c@PZ2x?8hjW) z0#UFDt`X2t6L>{L#1N4tYkc*pHH*|7suG*ohBWwp0|#G7d8+E_Eyw$7aLrFWg^$mC zA$K>!34UKxv~c&;(Jdg(l#YkeVj=FsaBX@H4UbaySE3sHcRk9`!U^9Y%Cb+)aSJ7Ol;dg*E2=w0!-(%LsvyGSJ!j337aM;u$ z2ylp!S3TFz)P!!U4dMq+N^gA^yDkBT&31x~rrZ+=r#cdlqmGxtI_BvR}=p}vC zZXLnzMo0>p8l!IkvZTVhC{zu zsYE_y6c#G@?oZuy@R|zCcpBDguI3ftEs0DVYF{>XMr`~9+7~gVfApQR-x(c$V!{V4 z)pNQUcO_K2x1)1lXoiO{MtiPKhzP|{J!ge16|3K zA$N9TS2YUjih0G`-=DF2@HGp&y0@$JK39JEwdc80HH|9B&#AXb5+jb|($z{ZS{U9x zY}ouU!B2+mT9n{jZ!a3L+6_6xt2cT5Sn1=4<-Wr0N3~=^V zZ5@^=?CGro2COSk(jn_oP+=OIn82nBk{m8y#ry>wLeLKUfi{F5@P917hZpB(N0AqY zxX{@Gs|o601fkEA@Q?MGwh_n~v?LR-A%ntOp}MlFO2|gwdFe@ma=}8=^*az}*psOL zBfy3>&(ZSo?h;U=#KkjU>-{!RI%RU`pgKi3pyc3elmR>g+HPJzRp2+ksVETg1u<8s zlGS)4pasFlxcsiUX(3WP;^&5@^PQs6Lb4&KU_E|E%hv20UhaTUkf*1Ea!8I?tzFv>Kv3c(4n3@p#n)t<@ zIJEdMxxDh`61-dF!XVd5Nd*wlLIUlC3-~BRy5Z_&aWV0kAF(z^Xup0g@u-Z!5?8DTmlbdmb}JGFgWu7s5a@o8<$uHZtb)()J0#XT`A%ICia@=DNX#3 zT?L2y?8gbbcI!&iC+lJqHqtr|$}qL2)NosV7AsQE)VWeg%Mn zOfMKSnW$9aesE5?4@I+j135&tisha>(=9t>vfqHNf3V*#K8Fme?s?%%PWm_q8T075 z^AviEwlJd{FZ^-UoHnV<%NMi)yMd@|xRB@5-PwN+eB6M*73MMs`}aReAcGk0M3o8x+;!GZs6t&eAGh zJMFEe{M<_RH7FXzrzDLWG|etN<#-&v?_N(`HEnAZBN|s^kU*M>Q_kVEIVB8CF<(7% ziuC(#I4i|`20A*P+6!l2p}F)rU`iI`hUqC2&|&~$)i=|7_N*OZsh3MElz)JFdxoh8 zC>IPtmH{rsh6@$GCZ&(#Ns(txNY__{h=`0yVIm98bat3V5_m|FCnlCoCm9HJNbHX@ zGzvI%k6yfBMKEPiQ9ZqOgR6QBUCm~1Jb!pK_cN!S;kOxb15z@y;iX)#hn{^>_r74h z^%Zj%Jjen9U&b>G!azeix31H)CYwPb-VEyPq3hYqa9)Rm#FTV&eh_zGd#iZMg)?}R z7@l7t8LozRAALm4wDYsGAHCSFhZ+kp&?p+qLlObC+f)+}jL6||LAf~1rWkM%sT*M* zukyrcaek=ZwYGk|+NnCjrT(J3lXG3Fx2x-q%Vywy4dg-xz~K?h-S(CQ!y|8giW2fN zQ915xr@reyld)W?fUANb@y)z~wJ#}!IGiPW4ZTZ()p_WjW!KNi!&BB?9>g|lt9~m! zq)YIF-16eyp26vb76*Q@)<4$$<^H`f%F+s()pl$e)oA~UP~R(#?ppK8?5j(Y!on!VJQ##3I;-=e zH96azIC<0gwp{Uk9Ww9ne6O3cJ32b1E`;%$Yxa$W-}P5 zcc@K92!Y!t-T{wxiF3-lB=Mw*HSY)7Fu2?s<-a$wprmlt7we^V^ryoStFh7P986~` z;1O7JPEGRyll$Mmdj;fwy@qBvybs2BL7}10t>?A7C2@2E&2i^wIm19zpE<14l%A8b zdFSpgX%(3;qPNvCR@d&IgO!P=tDkU;@BLxSDo*`X0q?Nd2v* zywT7Ez~oTac?Az9Z@?;2gB}|mGq)KJ3&EW7E$+~Fze(1 z9DusHlDnx)VDNNPfD|$c^v}S}|KJ(dMd~C={05!~MHtk7iif;k^=%TfQpH5`)l#)4J+PWxCq4k;m_Bvi7tA6@;3Z& z&fM54Ur=b|a!%`=a^s_dk+Bh&!m`5XvK5Ux%giN~UR8yG6sZnE_DqcUcQRg2(**b0 zt;L>QtA0*TAH|=1>xyO`c3QM5pG7Ynmq2>!9WLYj!Z=vb=^uoi&+|8yNLqS8ixeQv zK@ky6_>_u{jyEGFCT{V5VILG0NTJT@3~;Y!W&TcT;4J1T@!}I#NOGyJA2SiUR!F{O zZR`A{p|gl+uskb_&yAl__p^=HY~NK%n6b(?z}1~Mtyf+??43se(dAiK4R(Uqs%;;v zCfN0&?=tD)KF=BaYkBF`c=j_Dl_OV#l#G;XC0}R2G}xFo2_X~1s-qy#{4R^+v;YXM z0?Lu%yC4U`2jp#tg317ARao?ImMJ%O2})L>7Q=tMQjIt3>h4A;G5jZiQ&_#qS+_ajT-VNX-K@+-TN`F zpmW7n9AW@ArNDa4dgW(eKIZjk!sZPWSmV=VuwM!SV5|2k=4G;h3JD7)f0OaaxfWnY zBvzlnn^C@|SOR(!M<*vQ&dJM$+{q*9t`Jpt;T0e3b}%4K4=l&SB-d@q;X0IAT_-S3 z$~)QvtQp+VVY$|SYHEr-e(R{FF9X`Dn{V-?Ri*{)>|y^UQNWf0z$#MDoh10s#}~Dr zvqiQ5p-56_jJ*pBF6Ju(S?={Slw(aUmDcxwo*mk{J39730=G{nxB1GWhcXSn2ygI@ z(qH$UPDRaLL#aHREt%59GI+|_SnNoQgoLi}z)1oL8=D6d^8RHB$;smkP)HXSi{<9! zBh0$gqL1bWphP`Yod%jckYK}LA^?~tENMOH-K4ShaHCk24*00DIlbQLm%MJz<57d6Nb&-{WIGqx++u9q8Xg3vgjVw@dT1+5`_t1RJibk-QG4XBv&H4ae2% zx6P*xgN0(y0G7}(-YXeTZZxV9mY1qz?))n8Z42V62{#3v`caXt)|OF$El|#&!E5A7 zLlF}b!>A~Gd<|*YwtqFjCkF0m*Kw$-SW{8Z>n0{9cK^8yAlB*WDLaKE*Fi-JofA7|w?Pc_sV77qfZ?Fv49dMaGBPn1NXKMZvv@Q6O^Y>2mL(_vkKR*5WJhSBQ0 zR=4lSv+rj8kDZj>CA+hi4cK#rjW3f!y<)wI`6+sd`_&B0y~ls5bgmD7 z-P5yY;1VezDbfrbwPLJ&X#m6p8WEAYk&)CKsNlM~LN3~8QrbH;;*Ku1YDXIWHI%My zI*Y8X-w)9Er|}Sj=&U+3;`k)1dF6fBU8-1*tV3#DOsS_M4j5Bpa_SA2CAbM_O_{xF zhNoARx1PIfXt*7nsiY~0tQMk7pY)yFk$ZMf;+5{zSKTk%?~#ss^~?wEhM$Q9{T$w$ zed1FgDIp>zDjfXCb{=f%X6|-D9kt7shPiq0Lzfp?xx>`0zT`(1SX!@mcJ+M_-MDIo z{&(@*-X_tUEz>x8!Do!08AGe*YMW^Ifo1-Mm;Kp=qpP{3mzGDUe)K+sXA|>IhI>Ej$HpT8bk>N zpW*0Fq=)R1Z*|WKziS&FwBw?FX&|h+`Z+eTAe4s!F1)#Ls~qnzH6_Z7pnwNV=O;5=-87q;)-vOAkbJ))NZB~9ty3j!hBJ8 z8f#?w@sq8ieg*CqtB}L z7~GuB!Ua`yb^TEC?<~y27{sw zXf^@8s^zk9(;FictP%k)UBcB=yA4!5?5Xrg-Nx31b=}TxgWI;w^XG>PTd%tF!zo|9 z{3+>nAeJ4CwKiUY>;~2yk0%^0zdCxh)N4S?FOQS;zKW<|_${ckPtU}h`zR-b{;2sN zlxlbEb$}l#11W@_BJ`=IzO=6qzCenev+sW&8-@kreC+2>%}oM$uD??IGc&(jS$xT+ zASc;f>GvXA_R2~bcVR1LZebi;jMDIf0{^R;(vRX8=U*E=dh|rYFuMI`suI(ni^_L( zZhU+cKD?7Rzl5F=T`TB*JG^K-b9${O>3*cWP>*$K!}pB%#*Xfb14bEHSul*NtJ}R8 znjP=|N^^=slW~x#M35y9=9>=H%C=*vqvi$Kg_x1;-3=dct}ryW(BuC&Sa0KhtkBIC zB`p*DkXGpGfD83w!GU|!+A1v{WZyiD!XM@ze&-Ot75vbiMnxse)G0GUN=<^`n-bH2 z%!!I<^>Q4mjaA{)m@h%{8}mdhVjj|)ybOM*|32>2hj8~7cI=0HzkYJ>ROPz(Mtu5j za(4T8R$p?QU!d5ad%WSef6BYtHgTaK36&-9sFU$bSzpZcoqxj&&9P?71>uY4`*J91JYTN4xS3l(9xqRCHmYT^pAak6VjejY!T@r|N!n1dZ75;k@o`vfq z3WaxJ?;Y0Sq%LW5U;e+_kAT8G@9Y}=S9|$9{zh+=pSE5wCMm&UlmLCS0pA0Dtk)c! zt0M+YtJ?g+dUjpxMLX_wmO2El>>BKouU%;(2cdruDJ}Ckf%UPwF1N$Kvs`)OS);o8 ze_Q}s2C#2{S*a1396DfpP4uO$L-}7VoIXr(%2QnpQJ&|oPWWQ_G+!BLjK=Kt^d+8W zcFC!+soZyEV8m7`3j1JfX(E}Zb49*#-dUtS^w+m63G|^m(~Vr5&c&srkV}QXw&~zh z2>lFdH-_UsUq%*EXk4a+;}D=ur7Sz9HaTaH#h`d5QPdmQYkrj}gWE-qnc#+GU`;w~ zLE8(d@Cq@o?tB&sc7ISISx_iM)fX1?1p-O7inCs~`|Xt8h*9(ILl5306iujk@e{W2 zHEvTUuAD(E9o{4lbeUE~NDdfH2vI3yywAX{ZGO{g-1{R!i~pvnCk?N10(*|6MKaTl zRv0vRdDQXp`z$z{gk_c}lQcDX^ry)U?Vr(dEy3;cE?!{0jq3D7Yrj>m6?n&E#+YK3zeELaU zprgy{*-UOv6vOozVqqz^WuM>z9Ro~R9J|b7lUM%?M7p8?J${#Y%Zs_bBJ2z(p zBdC7c#{?S5T6T)!c&I^_G*BSZ&o@9S< zX2BeWk;y^jll1tojh?$KxnS!OmJ&E9m$>mXR5!${2;9|f*l%mY0Fmk2|W$Gu(U>!=u0}^SADtt&D_{EBKIwS4x@Rt7x#~n zzS_&THBd*kPBbt&-6xXHDzAxW-B;uL;^b@^#F-*!j{QqmkBcHD8F;q+D~VsfRt`-L z#_!w@QBZhcnJPx}G%c5UhwQn)OIjfkIX%Lvqc+MH=`&KiNiF=3^*vV^p*4;`$;X(Z zv2p%ZIGvE7$$w<&aB%3@j_x2-*i_n32>p!}ZrY(yPx zZ#mS@Ny4s1k*}(+66f)ee8psKDV$vV;P{lkH!=99)A19LO3}WRo zj-U0Z_}l}Z>81BZWI%2KAYE|E{wi$BwfpolKg#q3K{I4!<-xxR6Wv#5`^+5o z&B~X5fRr;b8wRNYATHU=)FC8a9!qx92AX@r=Jbec^*8Tf-L`KNU%1XCAva)w(M#+}FcgV`j z-dpyFL?oHX&fbJ%rN~N1M)r8GZ_n@lI*#W!(j)i!jO#kDb1;aCzYd&|mQLc4chRC$_F|$ta<~~oD1~07`7=*`UV14rvGoMHM$gd*PW+MOSUX^d>&?8l4US8hp zZ-kPhjU9=#G?zvdL-N}KW7-7u)IZfa#O3tqrb++{z3Y{wUNv!zY*1o;TTJmer?&kU z3DZog2AX6`d(d;4X6kLVaqpFFKG3>)gej-zm_hwmfho)9M@vj#_~kea?sW?HhOc@j z&mpW{USW)}b|2upNSySrz@A0QE%RjSB)F==s%tu|sx5Q2;EW)}wW5Bx<0Y^9el>qx zIKM_`rtne!BXh=UdNbXd$O``rhi4k;{2w9xrZDgPmd^#dAaID-)I1KbpG{g%&TNltv0&> zgKh)r%)W*@}vKCmwL-4u4=@TrDlNY6_}prFr_4GT7WG?*iLT zrY-puO;iDOs(^#W;Zj8BR5(-}TmsKc5`~#f29k`>rygTc=t*T-5O|B}4 zE2{}#L4R3OyCH)fHNeKg!6p1-iY$9JJxq;(_AZ~+hR2P5p6($$&Pc0%T@GC4CpeBa zE@m$H(`Fr3TJZ~wDtH~~UQTzDm{Yis7N#u&_zaLz_hVZ~AZzo?Iw_q=-#Ptft}5z8 zlsgFrr6=!lQmG?o&i?l9L$&XzW4|WeAgmeVl2-Z+s)V@G zQiKjDZu(w?C}Rw~2~0qHOmZ&`P*4Ap+3&7ku$q@Z-#0Zy(F*pbz^;TCdG2N-c98ED z7Gh{=Wx&&s$lLOx(G*Dx&jRzBf`TB(y25!6jA0n9?|k#w+J0kvhR_wDk5ilbSTUPm z*RQT$$Iye17mgklNK4%o)AN9I`73$F>L1rf+rj;go5l)xZ}dEZ4&t$su!lv8KHqs@ z$j7)oz&6|6QmUWc>a6sox3Y|d()a~bPsv$5Rmu_xe$IMTbHSNxKAMJE%Wty`ZHAZk zxvkF&%JgY>h?vfPj6K}B8qI56rZ`3<$W=U)-u2m*k5}++YW0;EYBlocRHKwT_VRrU z6-<4+jEt6~)Hkc8hN^Bl)y|{~m@@%JJ2LaG3cLdJYAp1=_`4UTIa|7OS}s4I7OO0K zuSpuJ_k^4}|9&r}vLo8z6IZ>ejUOtm3|Pk=SA2S-WZXAN+1TDxzrT~kxj}$c9sh}7 z4V%Ep$RCZ|z*ld@L(!r(7^U?2evbTRN*Kohy=8OQ+=9us_O5Wu==MZzNuQnKQvkl1 z5*1&pdv}K_tGlosVTS7EDZDp{FD+Hc5ePmGv(-D%dXEW~V^(cRSF7VjZw=pJe6?ha zlSKb&WrdH%aGZjMl7T4xt=5Uu*fOzsqcEr@Zr_jNt(^c^K9VRTu(tGGoaU-aCGV9TA~8kB_h- zTLcD!dFM3r7|l(nbffL!*k8&fS)e9%TFQCE#E^6rpvsy8`Q=0RoofUA{U5*}4M2^x z9j$<)0b{R~4C~kB)9hMvW46eaS489<2wk+a02_BvDd_lc zk_DE(<}5} zG+}FLB);@!Q$)APHyxA>foBg|xJ#1V8bKd9zwzXdbUEUN ztHQ=%534_u5^?IHZ(m(N_lf zmd`W{^yBhx;O!qIDR!={Pb&zK7S=Q+H3bzUYv<`Xub84Pm-%fg`8W77Yus)OD;Y*J zI(!$d=l9q8@YIp~;e2n?A%8NO$J>DLw2Q+hv=@J8`Tc*N3}y(@e8-_VrC5@t|1_{c>$jij9{$KPmjRiWj8+H^s-XY zVJEYMw7UAQ3L^-)e53sq_9i!MHCQD`G#XI6ax^mbq*-9{@6=+wASG|Faonz);g?z~wPE zI@$)jQ{8GSd|=^RgA!|Tv+ny!By;Wy@Ml59qit=7Z;%%}D&!Yh_ouKLc+R;4pAnGr ze`-5W-04bT6rp;QHJabm+Bd@riVD;2!}mZNz0^AKHnGHP=TF zkBN-|w5f1RO3YP3GsfHh>H}63W2%uN5Lr3fQpn zaWQe?Fl`>ssXB8=J+-{!5w1b8E;jXy>g!>v*#6Sh*E**+4!-)~3tK%4`a{;TlT4>! zq%}N8K9S&Z#L*{ohN<~k$K-zbJAPsgykoAslI>UiBbgn0N4qS}IZeU?)RW~&s3SkR z2Lh^>NpOy+br0_)AtohGby(A8U5ndMJS>FRRm1t-&Ji~2OltiDmKUN-LLs!&BIWkF zI|CKZ?}r``q3Y*@t{%d5H;fp!w;T;U2CM0J&S2K$(sf-2m{;TaL{FbS@eZbhGp08xK`IYVDjAqH ze_#k*?Q`H%-n_5>yXC|QY@t9gAPfgGEEibtl4oo85h@cr=vqt!VsZjbS3581&LRL9 z^52d&{_`XH_73of6X1K=r%Ob%Fx>_V7sNCHA|Zfhh%AcPO)$y_Ed)qV%z@=aN=oVs z%w-a63`F#ulan);E1qS+i*&$!kd7ki)Ud{c(Br{B46aRtSP7~Jz;!^G0feqF#Q3c7 zcSAG?_MV*n$qdcUzX^U%ANFVLnZ-X&KuZFTlrHM|+lA&6IEeqdIf3ga4jc?-#Ge$@ z>Zd;yB+dT*n(#en{Qe3?GVrD$&pRqQx*mQDT+Y60%ErhKAYNk7I_)mfU8K)7dhRxF z`CWta2{}-}vHUIwX<(+Pw*>_-h%%n8=6E7@U}=UP(7DoJJovRTuBjKnpd=J^ zLvPBUzeQ_gJ1zF|&kU#Wh|!j1Re`@tu1Ah_&Wl)&7ZKNq{j7H;-cNt`3ENs`+H7+N zi|4>>=F^u61@2U_6;26__|eJt$vKXGow3!%=^Ck)MXFidp#!Nf9Y*){Z>0!952WAS zcuYZAOEtm>m-1mddCSMT)-y-s-GlUl`mL6e!=1F&L2!_R`O8|B2r%WLia`A1KwA%n zxP^_4S)i4G#lXtR`Kq##sDm41JU|2fG&P0T6~G2U{7~$_aKML&^rQ`%BmO?`9@WJm z_l`1GeF!*QXJ%#9!)64932-7@?P%uTIl-3;LEaB}1M<}I9aai)8`3Z-6WU)VJ zYjbnCq&T;rrX~f63N0;VD{uJ4hHP^nXa`h)2*a$B0y!3L2s$lg+Lt3qR1O)L{`*GD3?w~e58oY?z zH)kSQ4YjG~P}055>NmyY62a8hX1Rb7bQYmP{BQK6Ry$tA(pyCsCHf{WpJQ?R_;rc> z$2C`;Pm{{hraWrJk0@4s;pq2|Z*6YI8WIWbXS>hVA@k6ARa0G~Y*+H}J9C>q-fZI2 z0@aKtJvl8YNmfBh>le$a-@auQu9OmsmAlj#Z>wt zCTFUtgzro3b0D;ZE4KOGV~eiHx!ltZy?<3ffpP=IGh{`R+GN57OGHI;S6zcKS5r<} z+X5Bm|62e}L)nbUoYH*_B_~%klUR5l+1D>H>0+A+E$>d`eczz2X!70Nyo>BKBk#z6 zl(L!H=x|!^2>AQ^4L2Xx>L2wgNVJ1(BI2z8C*M$(AW3wWB8vAh!$4I`ms#7IThV}9 zC@{m`>W7Z3zJx{=8Av$zuJGk8z*_BG-L(|Mt8L|z@A|@Ch z-UiV;6w`4q)Ph=olA5{@mK)-jk(fw?m~OyE8$YH7iEQ0$ysox(5(-?1>LIWA>3ks8 zKLY`F_LtSw$=|*i0r=Feq#Q5JcEj4i0WWRf@3|pe2jZFNFjEr=xhA}4hOk$JpR9m% zK^Me`bl{VNco%}l08nD~s?Yj$TG}Pg#wB05DBzJjkmC6>)7lx9H?O<%lEa67OA%$FRxvB%v^}m}QQyJs9hj#3lTx9dv$X7u zQ#N;3i~>E*?Z?Fu=Yt`OpoO{FpAoYedFx$PvloWE0E+2a#+5GFB<|5%}ImvjcCu z9=W@8wR?a4a?7to$*e4#bIY;Vhb3hEal|ki%;#G@HN}P_Y~_aDU?~;6cyXz&=ZT6m z@g+_nl`{8R-YDn8S!MOg)Gy}jZzQ=?iKndsjJ;8p%+1Yh1Gj-hwmrAypSE7!7>D@T zA)Q-so_{8g%bULSDU<%bjSqKi)a+(AWL^&L4e#$u*aQnFDk8{ZQr}Vt9vDod^3Y!@ z!9Ro50KzHI#Dc~k7%6y=wT_4sPDd_IuR*!K35fm6BYCPI1R#^r}1 z%#h$=y7jFYF_r+0$B>n>Jt@_=u99?)X#q;n!6O z1rYekpLO>6EvDEU)!2@E{jOjA0Oi;eI2r-H2ST0(&s4`hxu1Oy_~UM94G++{xZNSa z4#uir<^se~q(Ke{klgn2m5~};a8~(w89a@_2$~nFE|^Bar`J;``QgITymG1}4;xNE z2d{EW4cK0Hb**+8{Cr?(!Rk(-B&b5%5NKP}tp-+qptaR}La0Jr-43hi z-=f;eJJVwf+Qm%%^pmB{i@0m8`_(;?Hz?~6BNHUa$kmnL1h9vG$UpQ!IIQHYhjLskhh|2QP<`+!v{G^a@kE4E z0PdKvJUIrUu&^)|PEPT%5KZ2UrC7mlCZN8CqOq`=TjFGTq@a||`0SU?bfvk%7gs~D z6U!QsN5sW8DS4+$%t-ReJKtq?$Z3g(9)>_pJToJ4gtkN#1^(tp=;+jy=Vl>i=Eq0wSO8L zb5&VX9ItUv?HGTdqnmwA*ywXUPm-L6iku8DK1U^wL6=ssq>xD_VQk0m=8DG4{4}Na zUfAM#@mZ#4q&LJRMwl|0D{6~YO(gO>u&?V~oN+5TC!NH@8<$NjbW2SEoRO*g+5fFA z=Ib5Yil`q2`T$fW%akDKvqrw1=1)MGj zc^2UNbm1pVRUGicW8Pd1<;B%pD&w+~e>-^P_Tl(IGFm}=+H|@zfQOH7n#lyK1*jPo7Ib6G7&^$QWe=qlh!X4nv|k0ez_z};UMzT(IH2wI7B)JRUBAY7HxiLm9&{jq{%no zj+n4}m7l-x`5Iw3en}bjKswH1ONlK7MuWcXbo2UNxsnRWari>o?e+litE~Z;=LcF| zzvBNKKKV5B^zbI_xa?ItqHp_TffEfXv zHGFa{EiJn;txC>&|5xi_z^I>qo05jI;9YUnX*+|f>&}DkJxX#gGw9fCH&w@BBtc!% zPhM+~=O9yXeVXuj{3M1EC^@@kJjaSZk1^(wbveSUg1Qz(8}$EFJwMIV`+D3rQn9jm zGFAB|x-vqkTt`IOT-N;cyF6@Ok75%0<0@Q%ERr|WrY7{y^2v*TIneoInsvZ2#3yAm z=i4tPUPeBr2^V?Xg<*kyoG83)VMDmT4c)s4)@St^6~R`LaEP(o9OB8 zjwp4Zj|Yh$jNJGD-vY-)L^2+9l`H+hc}Z>Ub2wb#Vg>gHWLoUA??A?>FYDPKoEe{I2UJ| z$@*P-99kiYu*CjC+PV=ibA|kD#1ueV`^rbq)zrhjIyxE&7=|ayJoHmI9U2$tJD|V6 zJIS0I713n|G*Fds3(!;ANM z)Dd_L=0T87g%3^-Tn#@N{tYw(PF>)sUM1&Rf&>I0ggXPsR%P081!k21-XX_50L1XF zL(hVI11d$hM!-k|whcofc9E@GuL{y4T2)t?v;~cGdojLdRnTih6Lxy>$buU`W}e%{ z^Jj&R_^XhYe}1h0Ic_8rW!V*>3Qsio^-a=`eTk0c(VI|tY)Ai`rFjXJGpULPbT?$K z5iDj_Zd42kG?vC#C&d^`%qoEyt)Ofz_N(gsy(s;1W}$Y|N+>p{L&UUoP13x!4vH_s&!Fai<4u z=(GFV6VUL`U!o<2$*TWNG1sMkLz;5mx^0k3n~m8^-H{19v>-hxX2MQ1;V(lUOBAM_ zKt#wb%KPG(+v*;V?QgEUvsMeGyYuN*LkU84=oQR!U2&v2I`c<-ci_xI#F;?#0qvWD z0v4Q-oOEYD6)I;pe>M!mosQs+NO868l^yT9bPIxNIT7*#Vh3aac@%Km@It!{2n+=k zRl8H&69~RU%(!+S;LsUh%ZJ9sBm0}P>RQ*qK>!@}z?=mWrgP8%(9Uo7(5-?QI*dSa zO(B5lEBtTReyUCyN?^zl2&fDavH&s$=+vP)0F3geTd)ObsF|9Iv9Pd?pqZ9+cfSKn zA@CYPN&`^127`y+O4j2QL`;NG07Vr`p4q|?045|ap{0g~9njLXU%v`KpL3P#!!snx4zO6FXJF7y z^XINb2DB)^#+q~m5QA>?JJ3OJL8fax*WeCg6@Z)pPXm?|pc;|#FA*9)qO1kFU4AJW z%x?q)kmdUU(&$*(**n0VD)hE5no<%1Mh6E6gHI;CG`}tEZDLv)7RTE>#-uyC0q#n?D&zd06)POhSJC z`mLZbklyNcWyi}Z%$6d3tnI+oEORX6NNJKE&l-o1>dha~`zAgrmpix<6CM!sk@E0d zt6H=Opeio5#A+txsjegrXUNY9_H%-6*BNzB2J79(Z8b?;jG*0I;58(GD`vWvm>RUs z@X>%q-x)q|4alD#SiYB5Y-eKw?yZMtAk>HGBT%e%fLco)KtfRzK&K=RT?JzPSY!+HgYLh*Wd=`%?x=tS91JovwE*Y>-`b=YeHgWc zj(`OpoR84N{M^_;Q4@@Va~IHrzy~bc@&#gHK3#2n3%D;`UC5Kj%Zo<=v5^2eAX?-| zbu+Yvxwx&mPve{#>_p(z{YxY-vMpe_r~mF2++(nyqM>FY!n*>16vVy&Gz}6B_!US; z@6>W+MJm_kRCk&C$%+cC|Dh?&>;W7Ow@uraYfOvgYGBD3O@^?ScGz8guiZ zZGVos7GPXQ(5EL^az3vLw?BP$4{B}%fk5ed z03d^%2T15cI0m57-i5YtNkf4mV)qKfo+n!Hxb1{tC3efqALJhU@ZwQWho0vTw(^mM zmA9m*MV6Y#-nHqN3&&I(Yk5(gNd2oVA~Ycz%Z=e}lvvvZEjF{_&nskl?VJ0M%M!{I zY4T%7=|7|(Xgt<=c-XQ`KorItLOEOw2kyl2a+EZ*3}mr|RrjI>DsK2s${(Ijy~dIMx!BB3eZSVhbmVn1W%Z4|S~_AvJ`LN8 zmA+3ri5#SwL#xN@t2@2Nsx=3gulJ8e!w)KKdn9&N<_7Vv-b-ZUPL|decw887W_4$B zp^^$?zjln~y}E&Z$fRXQOl+0?y0`b{_=n>HJ+J#qY%&ikbxEZ}WyiZda`WR|F4LYx znRn8UeINd^ukQMbt-|&uI@+fob^Z=nWmy>5xI6{=Xtf}PEyWe2#5wpq2efmp4?l_T z4fJsZjlC6Ff$SJQKKvOn|;8}=^HF#J6 z2V`kwg|^yR3>>pEOrp;kn6R+k0<;HO&LNu2vwE#NEj5|L+ zXsrfAS2%N1c^-Hq*%>-hq{0}*7l56Qq(m@)H=L%Ut?ei!8ok@wi%Q=`v2UI7uOFOewls82NE zAVdi_#gY3@4qZ<(lrg_MtvCC9Ke{h2Cgz2$0y7yq@+JY0QxUseT}u-w%fQ*?d1fZ#7-Y_h+r6P-7 zE~Hg~sZrDJuqeoj{_zUb;V*f((F1(Q<%3zOFc}N;PdMlU%k6%y3%nT4#;vAObJV-6 z7j{a`(jS}~*cZ^V8Wv?nAvw>W$I-jLVHw4k6&&m5V)TpRR;G&W3yXk048(Wa zU&8Ef>XQ!gmcaF{WrwHcjE$DbP0^A=3h(Mer~>-C-h1t&zn?cN-y_#ptF1Qf@A$lk zx4B;E`MK?tX8^9R$Fb-EufA5$h`)>vO(@^Hl>y`0-BlgI?c+B-JZ_SnzH9adymhH( zk(^bHL80V^F^uX8_4nl-HVwAyXqQu-wr)(Hdm2$p-5t;FGVnW~87$v@K&>+&eg^2B#O;;$V7&;J{GG&}iTk1KmBp=}R92 zRqWWrgd7Yh0VZC2KkFpA!jaI_Bqp(w82<;djVfWRi2%@yRdJK4k+MGu_c6!$p`g1( zP4J`xUr?pjLu9$eP3GWII+Lr40;O_M5bD|QT0%}mlPw21$1<6!+54N#&&k_1N=lL* z9p99elnX4G;69U*Pu=+QVudvE=a!;nFZC}4{W9IFY_A9UidhBv!~CS%anAm%UheD; zX7W(7w#(0=4rfD01@SW|A_OA9{G9s}oP`WEwI_aTA7W$nu&}-rDqE?UcaZ8%d^Au? zcOo{>-TjG_St33;_+a~O3vqe<>LU*2Qmwv^yot1Hyjc&)U{U1-J)t8cMqEb-Q zp&B+E)7|Cy{_?l3IMv0F#d7m=N4@mL)jFaj#ZkGXR~z*GGN;= z`*Wr_ZM2DPWiT(+`1%`TKre-ic~p1HS^~7RKo>j@><{^W+SdEt6^myNwY9rFTXt!n zJb(ilF74KL>qwFQC-53LtPYoQ(2w(j4D)4G74O?w*O`b;`b1)R!42H7#gFsb}Dn|B;dR2or0l?*C)1#Zze2y1lnqI`gukyBILi zN32j?QG(3lOwVG5mU}#y_JkY5eSN}WaFx0jhB|mmiAqckR@8m(@ryHZ#EE6Dj1+Ed zdd$o=ZciLyd(@57c<`7zkzmrY6Ejuj6P$0)tZ$g(>KJ;WSvtLW;0^DR1%^1>#1!Z8ga9uiCU7JS*z0mEyIpPWq`))HOTdlJk@R@|CZqv3#*m2kIzc-B zjVak{u}MGYp8N_z72(j#Ra9?&94Sjp(N^Lh*hbZONYi8c-?rjaYqBSwnVN6Q5pk!v zH2(23lT}P_lI%l4rik}j3VbSNnMzu6M|zLK?mO=Qd;Q-_`IZ)7O&AqV92H-%rlb-l z@w`>=lbTHYs|%VqtSQl^#q=%dM9vRoMq>Naqqt?GrRkD1&T}B)&P8QF_8s!c?PBZxb9D?BwDE~yVKk%Q{)ju<8Jy_ zZgFr_nvyKXVTPCA{NFtxoq`1NuFRp#&E!O$wlaAVf`DVik$7|_KR?aa??}u!L>9@a$Q;a@1HhSB2 zM>o23RQ`m=o%1>hC$@-JxjWC^m_>U10co+EmHP8M%RByV9ZR*TL;@c1qt_!Uz1L}1 zw8$Ske8>t_zTeOw!6-LS=r zYV>DSVKV5I`R0Z@g`q4yJu|j+Gu>r7(dpV%*jc{_udP$-ej>O=z$+jk!<$i9-TeH4 zb4*o~oqR(zV$9{>8QJkoVYDe6Tkp#I{ew9%4B1#e- z;Fj%e!CAQt;U$1?h3$?Y&^$J_&z4(}b`V_XSUo@{SV0+Ek$VT$lE3ut$d+fcsXB#_G zT{Rc^vWQYiJ#mw3&zb(ZW~2f)F^@&ot?TS)O#;uO?amAyBnOX_n`I`f);#=OUD8cuJU%s`c`C{^qHhH>1O*jk3554nXNI7qRMT<7&#(JENi}h+-C)pLHoRvWTaP2 zV2e>fBDdx931I4*SL%l3L+@=N0l+6i>x%?u1M(Y&a9|Yyeu0b%pqF@sS(%lY2`H)Z zg&K^DV8|p2oK@gk#GLr>LTQkd;D}tk%p}vl+hO1U{52wc+FvFntSmj+27Y!q8gdQt9>uu zloGJ$KwoIQ8A3{*5|U(+$=dC97w^MD(YIHlHr|cP2R%03Nq3SQ6Dl0OllO!#Zgb|H zd>v^%*nOHivfE^SQ#Kl3`dOG6mSjr>aWu+|HpBcX`jV)MZ$rzo=TBU1gemw%!rO#m z=pzG>uQr}}eD~e>ee!JutVG$2q_4t*oaymBdO;ky6xOw+C5N|yYkVXN$2}7i89v6v zk#7q{mHhd9LvRz-V^zmj*FaFr9{zjSqdlL#?OEQ+8Qnw8hw`>Z4rKya3*2|Oh42-H z&0-sNu3TZx1i%d1Aq2rX`a#|*w30!cvpfw(!vK?9Y|~w|!KV@Dl^VNxn9N{kweD6| z&oNN!M^;6TUA&BbOFE@uxb(6#eT%(l-zWM8x5Mv)T~COH_U{+*G+vXo{%$(@cbf6j zgXhHd-`x{Jgw?7#!?-L1jB5XIdy56xxYN&MXMF!_L4Aix#-o_mQBL}!W#8}Lsd9nT zS&``XX$8z_EuAecxK&L4eQ^tC7LAD=qTOAf#gHzX5)a2*UNx)xFpr@fo+gVEUriC- z70Pn225*CjVz=_$mf^j*=kfx5th^oyTS;+UBL`B4x9`{Z=!88NTBek{K*~ zJmgD_8Q_i{1k?cnU1_Q7s2n*M=ZbWrSB;0YP|&b8pTw zm(Od%YgE)iNi|FZ%x8FZnPKJY*g?+i=vqMz-P)tigqcYE_P1oC@3<*RO}djB(|OBX zMj!wBUY7X%R*(s5j>mLbrq^9sN3!V%t1UnZ_H+os;9Z&4E%9>9GI#h>EfdT4L5NAgXgy01Gs=*pEK zLs%tv7@~jVj#*uY1f0IXTQT6WYF4T5^1)!w&BMe~$fdzG?%iP4za~!=$ zh;!;nXl>s69=!M6x~ds8w;3u7*yvRp0UWb-nW=rl86^k4RX(YTC>Ra^y9p{WNV!~m zKf#Z%Z($|_$~!P1jQ=bN1}s2U0FO8DMF6@0kPm^k;!KFFExOR|T|%MpKQFAqx2eD&V!M(>kYdz~lDGEbOa zmk{fFtJ8X)8OT=S%!$T)`gZl{@XNBZu~AJ-e6+S7QVB~bUX4jo_*f6FW6IBDZc>eH zJ|qnVzB&Y50GyKXJ^ri5-g$DfF}@nj|KI285vU-HMb<#!tZAj(mkAk!)ReK~znFS{ z-K3yl$i^m&If|k0V-4#vdsWkft`m1dJ*8r1BTW6@s>PF#22N-vIAD!{edk9=E%ZdB zWB{Rwz(WGY%@ahQke4?y;=j-LkHk7-|89FJ`99dy#6t)|{aR5L!nI5{|J5)IVI@eg zAa7h37_KC%FxftSjF_>2krfvg6LBO^cRumsBP#gojdQs~^lGA)6hwDHSpZU|?BIWU5Q8S1DFDFB>6J5EC7+fu>w{)vG!Vby#40(-@@*Hq1p zGic2a>I(BAV1$C-L)oM=2YhDj@PWg?6-YurU;uJ1jF14lPeReu8Y)!lFeLU0RmV@f zf4TLtfW^(ttpaH<|It(j_TG-p!qcfW?6G^Kml|JU2*IGd889Ks!MM|P%rygKQ*d`;{l--y^<6mgq_ z0F1c-1T6y|BSOJ~OxHoH-4Ew3HLI){DqxcFZ@L236`=iK8SlAz^(uHg!pCbdV^uxP z!phoq1DAN@iTT0FgQ$m&s`5d5v|p9Ks$T-*bHw^MZv-sT2B&Hv16NKRKru>D5dQ(^ z0kC1&m!pI4flL!3ldsZd2xoA#r?u-h}CaW34_cp6-8U_I!T)#E}$qZ5q_K6wUsf=FX&=hSFLZ z>=hhI_|k-<8Bf`jIEpl9`Bmjcle%L?F_kQzFHwf2%a4)dLxKh@3$lSh_D6!=wc8*S zfS1Qt9t&%4u<*hHLivE(BW63Dx`#UvsZEI+^sLJ)qoyplfW@3zbnzNPK_t_l)zvXs z0*uR-gKZ(y39{UjfqWmEWGb>ibP# zMt1;TcGR5)yw@2L0SE^U4#FLWa%dp?v0MGiTkkvzJYLOm&y>SS2O~hFCJnea2+%Mz z6t}txnSifp|zt>0N zSfVig=(EQCZG_{BFmw$3kCrn|xQ}zKtJ@OK?DvGL-hR&8pkk`tl-;LwL>S*-unU_6 z&;e_49az1P0RVqqbQWS7oa#-TfbrOU$uoXQV8pL&Y#>sN?58UkFd-`jN)DIWu zCrHrrzezM`P??*qC5zvGSv?5zVlWrmm4K5*FEz3Y93@)y^D4>A%xBZiS!b=vv6< zXR4EUGn1hI28jOUjs&oLa2gy!`O0vUdlI2qaZi8`tK>aenv>q(PF2T%svSt4c-0ncs zH!l}`bjhc{{-3w+;o{eTi!=owo3%}?;E>dLgxJSJ@ku<{_LFW9aK6IzrV$@ z5-A8qN7#_+%DMisBOD5M6ob(`cmmw{O#!N>MgKQtWz+|cr2Qo@g`Fs*lPVL|l6Ip;jJko#LC>gg2(fDtW5O>v zr_0`BUp47xAKvh`_(J;(=rI2gC&9*}Z)9YosW`q641&ksp~1luQ$N8PM}z`91Nai+ z+EbTK&iKN0`ErCj!=9Vx54Yv{zS~l%_?5$pPviLST)!0Cvk0^Cp^Y!UM8M2gir(aF zgRKRK8uRn>DM!gX>80*2$}AR9Pz(|RGz`b9HF%4d_y3NOC&YjOpk9lwFdXeYDVCmv zU%D)2RZc!IlvLK;jv;V6THL#`xFk9l!91mN2pj=K)9V7~2v8KRCG^b$vkyBcp|2g% zdQLYV2bcr7!8|*+XXpF^J}(^~Lz=$0hGMoru5-WnUKRayi-)(+G%x7Q7RpMfUbH9B<@#LyHiL6nA|i$( zyD~lBiHkI@Kc;Saew5x0!!Ao0dajIXLpot}`M zIFcS4T_B5dQ8&YOtFW!DjpuLUeXv<{t(d)Eb|qnQJ*SB@4I13YBfA)(A6Id4t3(XP zBMK0zTSJ5Q%y(wU6UZ7;lTm;8om*XgG|n$Zh(c*To-M<4*#y;g9>{$OM{99z*yP^# zyQETB>LF1E>^ln10!6E)be16yIRixyA`=2bXgcuypvMN+K-WITG{WdE-d6+20B{#D znuj3Sq9^?5RA;@%*JSiqUJwY@^ovsjp)yG9_Yw&7=1bp{BLx;vs!&RlXn%mI&J##1 zr%pjZ0fT(Y8X6`l1!zQo!UbdkuzY>eO=%5I++a@p`LirP70u$uSXl7RH0QHz;EvwrX}028M+3J%GPSC!wH)F?#1&cnxtpp!E6K z5C>kGus-3if=M`p)hxn)$O4Cngp1iE)W;;RqvT(L2MSA6hL5|~X=xnmxUE1|qNT8g zJnq8`34ttp!%4nZYhyi?Qy*s!pN=`?_ZBQ=eieve|-5X9^OanBzUv4+kZ;2l>Ufk!C=7cFufk2do%O% zgF7^a_cacrr$=#%3{B6=Rhx6d?^ln611 zw}o;XIYZFV`mIK83XYhUrHbGdurOq)fJ+*1l=SMH48JJ-j zplcf@xkV5v{{c39;^6GxEe+AEh};i(Tffl5(p*}3`lr;&!xIl^HFO@`gbl`cRn_7a%PIBq|_) zA!=19`9LQk57dw<4j6N{t6Z9gAo`_TpWg_J5#iIqIt1+-7>XkL)!v_kAPD4zDA)3q zV>=ZllH1LD#I2ASy9UD3-IXD5S9XC*Zxkl)K#_vg<-2KbJYey_&C_!iV4Uq*%{*>hLX#C-C&(;C3jBpXjftCa~Gg0Gfom&UHAcI2wH6*yH zg1Iu1n#0M-3FW}ftJ{8&AW#7J24HRV&;?LyX%S%{`3~d2v;@I;5XS%#?E3am9ZY3D z02ddrY6ns#LeuVnED3};RpU%RBhm?!M}%qx@h%Q?4bSVo`)9oiI9o>^11=8N5%B{= zm8E>U8NAS5L1bjzH}7cRs{!-y1ITDuK}bLwzYD)@7usbc@dn;8zM?@bXao_y8^Rd9 zeLEg_2#ARV;-G>MwjzuoH~R`3U_?Z(vr5;b8(-2ID$ZhwF=qM z;~yNMJ_YJb$EJrh=LA2Ip`jr>)$7l0`z~D7Q-1g=5d|+SakS&GaJ2%^7N@-Uumau? z#j6cUvXi7KFUZ9+q_`hR{8pNmy3w@%ZDAvkM?S6t>m=Xj5*tApf!66|mbO3@C3~I% zv$eRJxV$QaHYwTlB?03PJv8rj{49;4k5TwuuesqaphOUfHqyigZX(St zzBe(IE-4Wwbbgl7>c!KKr|yb7PcOtOdhwbS%fUY`GRfeH-Sgp}XblD9!lca98bm~d za)fed>X+0BnefloZv`70Ue=l%tx(OgLR1=X#ti zDrrW2u9qx1VwOui!MvcTvpbvNDp~S^Ccn@r#;a;YEByy%E6eXGvawZ|_^MYm0k~RQ zvty=>UbsPpB8z=spfB&T#xcnEAi|gFW}Hnu7{>Fk-?JuO6X>f_0Z{??HbIk^|KMi| zPJg{YIf6r<4GTgvkvA@oBiFP5{P z3@@DSjsVTA3IK;;$qQeA42sV5zF-EHzi6+0b0VIsO#(imA{0%Gidu^$rOFo1T{Hu%J|^?_60#N#*4bT zG-x1^z$d6j@4v$FZ_~Tl@;bovT(s_O#vf?uaDi;_`tRU%U5JKX6Jf61#Cst8O zNhBa^F&d2^e}`3_26{_GtqC?Ot*v>8@CPosT_kz|bduW$0T_9*@Oa6^e{*^E@XgG37rGNNk*RU^6dahy;R6Ofky@cKjP|jDi+#-m~8-E8Z>NI zn}AND*IEZ%bx~1~N&<@WFqiBeBmgG&@iX;~Ly``>)Vn~}10G6pTmVq%03fTLme8($ zAOHLI=%}HGg$7SS>vi`+oPLvQcFq*S7v`y|%BcKTSHpry9`nryuf(Y&$m(1t_IvyK z=Jy3*)yH_#X_w@JZ&}y0zOP?Eo2)Hyrxhr6Map89Ty>D^~@^f%-aIzO63QQ$H? zb$mw`psQ*tnv5G2qnT{EcM*3yO7+xOhg4s8?q|0n-N|c$E6GiJZQPZG0_J!^ip_URGtIm6tXL@9bg_`#~MSiCc z++!_OdpHrveCPO#DmW0edD&%!Oi#$XJei?xt?+$p8yMtT7coT=G4WgiB3#N)X6*P{ z0SwQceOEvc7r`F@jmR#gf^noN3)J#}xGcVS@!}a`!vn11Pvhg@S8^F*x-3B#M9K5F z<~*Wf!31s_#F8Iq^*bf^{i0w1G}!y`r0*UatKSd11%dRs3LD|y^$a!?h?MQI-}<9$ z_b+;o01g)*tbYjTfVB!iAqR@nDz77TyPtK24`5u2012G2V zOo35^I}!>JS}iR^2JVH3nV|#%1u|`)MH0{=Bd}<_o_UE$EZhO#p@c_H z^`YNSR|JtZ_7v(#q=iAYwlCE5MWEt@TVdKdFK3i(8nSC)sqF3VyF#O~1GE;vnI8Wm z(F0ZnWe2E&VZ^Qu@fgsQDnlm#t@|t3gJ5U@+LjJLMf94yct9ozXbW(NlW~MVR#Of1 zc?evo;?U60==jkU{dA{~IZ#un4oWommGHulC=|dRNem|uA3V72sG-LOQYf4`!KI~e zc~{#_uu)M_Rb7ceEW{A>7=FCt3%7;vB ztzaA99Y@mZX5rPdpTSP1MceJpb0P+Lg(^ieDwT)twWx8DEaFJ0_jyEU(zD%G&=sRJtLD@ztxw2pTr38R25s`UQoQ>k&Gg&#h2*HtGe30OCNM^x1D z$q1#g6=TvXm+nqZfvHq>@uT+v;YqN?Is zGF#3`mbF}^%5T#xZ@pQnQJ}*~82sq^NWPCVm_G@Xyx!MiSN;EJIt!>Q+wALufJ&!y zNr^~zNq2W6jerW$p>(%|bc2)v3J3xU(jXm*l%yb15)xA1x!?KztTk(9&CL5KaK*XK z+55N0=BJ0H0=fF-LTnj$IEb&QPv~T(^w`4B{qyHV5+GCjK}Za{&)3DE;%8D`F$j$< zLVA!0bR?#T{-%t=!cxkYYGRZWrp5Zc=W`hR?@APS2ha^kKC{KHu+dU;dDVrT@buup z!Da^LJa*HDIOGfm-|+!pD8i985YpTOah%Iw%oV?VUFPeW)+{K4NZ@tA@W=7Sx!mhP z4Cy`x8}OXazbyO%Hv(Wj;Z4$StAR+Gkww08DAbcM#RSUTCO}jmzRVBs*!r)b_cP5D zJR~cCFR_7&3AQ6tl&~+M4Fwd&^2*9(cxPLm9c?4e3y2*|LTteBf`Ya^#9BOuHY{}d z_|;{B3YUPCQ;^FFbt80*p#DKZ5nv%hxG}P%;TpRGe5DB&4pLPEQ(DJ-c(_-8SjIvg z+7ALRJYgHa^r8%|!mqClZ3G|IJh0-;LZKKr3MnYFXrG3s>IJ?!pM-uZBw) zVGuw9w*!AnsUxWV)tngRwS8c0hTzUj>Z9Nr@L%5LiR=XO4WYI4?fn3ueg_nDqET(_ z>&@I`Bd0#hjcUBB>!mqr#6#)1M{!SEsvVg=&9k<)9+r?!r+=QLLNlj?MXt$ASmlLY zcdG+cN--XN^kwm8BW4IlVB4MEQ@WFd@{mgtp@mlb`)H{nNYe(k@QFD{HdVq z5t)aR#cN|ZQr%Z|VM%Wye-qsk93TvJq}d`(in)f3DKGm~=jSVH%=oA0PoL6n(yO7j z$agpqC3WdJaw-$O&5(0afqa^olxR7f;wG-1;Rf#9AHtPhK8ri9S><=@cpLjvT&pQ` z)6@uF8#9gE)zyF8NvAD5N@MlV6x~9Y*`R!bn_OxC3$0{QSWgI6i*~>O{Vi5PdutXb zJHKNF4eeMu>vL373x%Rxq^6Awn~_P5UuI*RndLlb|Y$Fb-=aNHZ)plf5BV^d|X>MInzTG@F0p^De5?l9@-@7YCP zZCvM40qF6O+6swJgVTQha6;34!T;D5s*Odjmbx96y6Nu?PpIHg0s>h(w>MW*^pj`&LvYE107(Sl}uH%RQWP>1J-2h!+}kbZf8-mmyTk7F--=zCT}P zx+mb(V}^V_KRtlM86IFL@ZN^fA2H8Ey#mfFp!iTopLi_dk?ZcWfr$T6byEa9<%$y} zz94)@2zWdF*4Ww&cCExmw;|_d83eAy@LeFib%cKeLns266+C&eU2?_==XXpD7P6`h z+y3bzCDZ?s>TKYaI-iidAVzpYv1D8lJ8#~=lr$nTvI%y^JCmvvl>k&~;io9(Qx!Ee zBW5meOC!`7-V0W89ANw*KNBdAnL+8&PYrk=wMxG6@$n9u#QO>mxeWJtu3n36^V)St z8_gbyuoSfMxp=z5T zNkHvBPe~M(L3CW`o!P%pM9GX~c0Tk1lA0`|Mbb~#)J?4AN#r3XB_89!^%dl`a$Q$0 zX|}7G&s|#qyoaXL0qUu7{#<8TBz#+rE%%o4byuX<=dVrI%?-lV&=kd*9W=h}nWOWMsZRHMTVv9J=^E z%|wS}!2>^26>f?bSe8`)ngAIs7@m+ABRI#5(dfWE`+w8=UH{!@$co`GZFtTk7F`rI zWM2$cL?GBe^Of(46fY9tKz#i8eC{1_G&)z?IWJt`rRC)}gTSBxNKr~A*!sEVrrUVgm}=5^@= zG!GiP^2k-K<0W5J9Cn(qPF+)dyV_g5$@lQ333T?z&1G@_6<8jXY#Zvogu4hFFOcZN zPufly_i+T5n4DYywzUx@Es$xp~tk2bw&dI@&%bX5k_%;+&b(Egm`> zPs+12rnGX>y>RR5IC3yw+?4q(YRJF()-3?fj&{J+B3+rNoV~$qN1O~6JWqAkQ?s9R zks3ug`t!J_+07-AvD~J_SqH>G*}_h6XFk{!5Za$FDCge zbLFNefLgm+61$um>tNmvRVFVgs`>=tGM%o)sF}+4Jbwig%MD3Uv~H*OwUD-og#`$Z z0V-lMKo{BU*nz1Mo|9m2b^A4X8!hO4ug_@s$fFA;$=W0xOWbg;%>|roi=Y4L%yHvmeDQx; z0RN+D#7y?cKs{A6#78_~VOMd=vn%AeS_mU?*r?}A+k;CrgI?D*{-efD3yYvUX@U&QDYKs|!KH|$TbQg+**WP1zef#EPWc0hkp6KonkM1w-gSh~%n9|~$&*3&PMSs9+ z8eLSxfH?W#!3o+$IF3hw7`QHZDGVdBD-jF?Q% zj(5S#4!NS^n7#6lR5AxB10(jdxbOc!G2sH4xgY>}W7g!RW?TWmVdL;FhSC})a{wqn zlqjJ5Q}Fh761W7JfDagcA(V0g+f%>FpS6h13gp8s4T&1dnC@*fhNJ>|hr_;_v+& zSZuv|CTvxXwe~}lc|OaSu%rQRJvlk~78Wq{t3`(4y~`si1#e#nA&r-!|J-&l+h&J8 zr3fy2Iq0bO6P;lnHz>YN~uMgJENF2lyLgwfr`vj2_5^eLRs6Qy z<&u_iTPsbdauZJJG7@q4L-^mYZDLSMsh{mFXY@7NvbZ5}0Zel+lTsT*-CGG|vNtaI z{^ZiA$cGP(sp+X(KZpELpWkL)!6LEh7=rTtmEg>c<>%jT4LN-J zCDkU`zjMTWk819Y{`G`*k|X&H*X{Q%I$$l-Vh@Si2)E6?c^kl8~gsvyV zp}i}J8T;eqzs>sh@8Lc7=j;NG^ryg}+(#VGufM{(2YKiK%;25xfju}?;n|P8^8P>U zF9m+>=^*%h2YJ@eaD#swKr9Hg93h(h8I{F<2mBT2-%NjcCIk|qy6Hh!khmDp-(g;A4@MiEL1q?7%XHjS2DgTCj|O zrlk(bP$(2K)L77^7lFdQ~5L2WD?V|mk1HF#@VDX1tiinrMjI$C6ofq;kPiX>#%UH{P@TU_W zX{k0mo(_R0>Cd0g?CL&lJ-%Q4abE>aCXI5ui0Q|WU{o?guiU3E5Jj_FSQLU1Ln^(! zt5}`kzIB5JuVaO)WB9_>;O6?z&vuWjq;WHPZcR$%DFg*dmEqiUwi;DM-)JRhSxDXcN=ZU<`1LdGJ7j6U`0^W~3d8g5Wrze+UXa zNMZ)urD5y-Erc*vim+uMGf`wEjt@{M&=4P+nCMJ|qc6$#aH&-rLdX%ABhjVVJE!BoI0krh>1xmwGp|#r3;0CSLL`+s{T;*h8 zXjlR3hPrtB9USh^VU(y=(}%n9>iFu?0<=6(#u+Dyz|iskqfS{S4Monj>YAW^gj8S$ zgAQ3e$g+m^68t^Qo^8K}4pRIqsCA+JE)JBGE=W{_IU1BF@GC<&qtI=#;kDF22eB(~ z#^#+z(4FM6(WaLb;*ZOAOc^4^KVE$gWf~*@xX=06pUk@A6a|mU%m**$nZh>|%phKI zsqxwP>R0FNMPH*|*>cLL4t&Sl(dJ!(cD&nKQN%@sc`CXSQY6GbqYg9n{39oS9mWAESIBGZ2}iDjt0zC^x4m3opj`K4u=-Y1Gi`>B58yJ9N}HUCj$uzb|=*5J?vO2l7Y~=$a^=U4uL4t zcn_~JhkT82bmrLezhG=7dnq>=%S*S>^g_A+Hr}p@I zzbaLVYHkUvs#4*(FFm&PChXg`5i)q+KZUBo3rzaQWL z1f=G@6mf?`h33%1lK?aFCdNfQ^&CXIl$1OsHOTg#V* z*?<3~$YdhZ4};-og$dB;cDeq=08gl+;HS3c2a@J@VU+U_A?=>Gs7=QXda&oD;k&i8d_GHhO*MN9XEN z@!?v}{h9o-_^$00O-EGNau$6>P43J1TGN}Moa~rZSr8GC=oAT)5$LCBludy8!}h;c z&qSY=(rqF-$*3geVDZSfZhirXwEGIL2BI%MH!4`MQdhG|Iik4dK_IeAjWf2b8n<^9 z!!Y^MxPtA}hhbY$mIPhJZ751c1^Y2S4756}UW%7tuC)016&5lMex{EmWZ*e53YAHT zPOdeEtD;$KhJyMRV^Y+_*RcrJn87UFXr_1?QKw23aZ;+Q3o2^*wt}ehjw0QF<-#7> zLvQaD!zZc3Y>d5ZG%5B;c{pPeW8GSMz2;^b6}{bu`kZ6Y_{U?@>xyn8z9d;P@w@Mwq*^VpA$)I<-+O}Y)cXqgD${XH{T#XmoIw z>t5FpA+L%jNBdI?p--*oO|O2Etc!4CNg&mmTBqSprU!B*o1y@|L*y&aW`^ACi{pid zFk=&@PM~-g>LU)CZ2Cw%8f$1vLS5$C@g_Z`kSnUF51QXdh2Y>CUXplbb(%%>*1x2`mLsjelVa;iPi z-aCvx{9mkU;rpHcIhX!&_p_N%(#(3FRGz4>mKG>|BE`OHsLM_z&@u&g8rxsLZnM@I zfeE{+-?3-|2jiA?{~niT3cJ>7Uh`IHSP>tAl6_zwbFfTfW(to&vKGHreVF4$Y`iJG z#Uih3kJ%kO;GabhhUqo&rm!cATi=qE8QVDO&eXT$QE|@IAg-GH<#s)SDNZ`CUGL)o zx|vARU{%BNpzFl^bOL=4UvE6CigBwa+uq3uy)tiRL1Rk_wKwM)1kPw1?s6 zCG^{?D&7#4#meyl>oVY5beCKv6H4%BZrLBwF{%R^{mM$u9q1B&?78PrN%ud>(w)N6hPB#x$}ozbzF*pO)c1-TqpO%L@`icQ5n)u4%4jjmIA% zLvP@{eYSq1zUxb7}`bWLH-QcuR5X6HK?bD9!gpAx`u zKf&f_%DB0(PO&YE1aVoNjX3m?N`gMp_xBAym+KEH?U5$Z`qwgc{RW}C`)9JC^@Xr0 zuOP`(EHk;47u0u;cj`OK(#pbS)K}&QRUD#>Hh0)jlP7TuaQ`Xc{k0zE(tJ5fIW)$v zj579EzFq`!Ky{W6{P@v1N5cF;CWBK4qZ1XJo+I2&^y}9rWQD7j2N+5?Q8;HOe5gB*^X>)xS=xO8XD5G$juyzBAAu z3!(B(^Nu(^goi~DPrfxsO&xvqEl`zZV{2Hey1aAo( z7@uv(A3pI7Qf%0w&-h!HQ)07A8?kS=mw)9i)_8e%-T5Nl!hJSTLQ~sEGVtsm6=Vf7e3ATE7OKYxNjtPA3NEVl7dcL}F9c{*TWN0^KiYOv{Y| zaVS_P;ZQ%@yKoY02#gdKwExwH=Kh1kXWBu}J^K82< zubteiA|qKUL-Sl@ubm&dX90eQmCu^M$)=JchXu;DrA|~L;}ARSrHOm=*`qDh#n&UX zuAvb(DC3?f|MSz;HCDDBWMlmMEB1k09Y+6}sQ8+SoHk>y@-aN5A{<)UPg-rk-)Q-J z%=4!F9ReLPT}rZjk}4v7f!xcbuCP~>v2$yVU$-*wmX+f#%n6$#iB@=BnY1k*G~qU` zqzohwhDGbTce=hTsierg^mx|ua4}FJsLN9HPU>`Jsay2WH$hw=JrM`|L0 zNA90RPC>~_Q5kJ!_jM9R?4Y~pzyI<53H!13j-w^IizTCP6EmyvGTLJ9l50l z2kv@i#-+!?4(1<}^#oxuqN@}Vn)qMa{mG-8`~*7r^9Cyhb!)3so8s* zqcJCCji@l#6fyNKKS~<9NxHf8j0TtZ(Sj`Uo~kfDlL1A`gPP;|9sKLYYoGqR>uK%e z5}7j9mCS>Z#fg_md%iC8cG#J=*m1J5Lro1e((jLwI?jldghKjlb zHKbYCgXboB;yWt^y!168^08Dsj((nzi3PeL^L;(8(=)fyYsCc z6P;Kv5<7OMxJ4+alD1P!BO~u99RJhM>=5Q$9)p|vgcUhOVp)`UdQa=Nm=;m=)9z7}sk zZHAsJK$2Geh0-f$h}+$Cq`_CF7nuxgaHq4SnxK`SKYVtFHVyL{BKprVYEkDlCV0Y0 z(8b8FEbPY;U0Ue#ObO46=7+jE(dNmE#RQJ77JPf$RA@925y#EqXjtH=gLGiy48BFmP?5r!#ao>gY6iBulDd*|td08D)uv9y2IVTdtRT@jY)iSs4{)=gmk|5FF zXgorR!zUbm$CR7xna$e!q0fsc8J0cP+SmX2{o37@_+27Kg0o1o)obsa!@$ht!7y*` zaih4ePJC4q&mu)xaLG+X>F~HfzV53z>5JM_&e-Gn2?}?&HD)rMhxu79Ii*bGPEWe; z8TEULaSue!F1m^GV2Zx0ZphqQ8znxuxUDDMW&NHuuiz$8ZGRkl1^RhA;d&1y`P47U z%LYNeWQKSVXX>=hP+=s5*qY(nd~mhB>8sac2gBInEyXlro_H57muZ1lof0$6*SUuz z(y#JA2_+_0wjN*QKg{-LN$Iz}Lxm}Z8H-9tAl)@z`N3dwy!m+cN5g~kh22@dz^O;G zyH!<3rn57TOkG|#R_QcZZC(UVn*yZAJs zR??dr^v&+~V5RgN^5oaoMEZPf_19=A z(k~{7l(R%vwlAOCzzQ@I-qT>@zRJ8Gpfl>B4F@`QY} zLm!+t4pd9?)m^UQTWj4H_*8aoq%cKUfv7w=_n04zj`iX)h@=uQl#Nwfa1b$?6NnxOTPrOJ}0(cwu}(<^NhUoqkFtIPJ)9K*65@_mg&f ztCIbCp97hW3+^2)KBkQ{x!@qf_Uar_G#P!q=`m{N4{ADxD zEcCX%^qPD~T}^ZBtmN~^n)}BrF4avTO~k`JE~n1y@TCCccdwbUOd@oc&D&$jCx>+| z30G~yfC-SuIgriiMZkn%`T*0U9AjcqI!G=pN+%=qZOxMp1h;Q;2T%`Y2=9~V#}m6$ z%kFNg1uDAvC81#`{XjVi!CIBFa?_4;#8Gd==s#a$)Cj~q4CbbGwo0zXjpYUJB8+lZ zGlutn8sX%v7_+>6OKy%cQNmG@T!9Czl2!Y=tmD7lEOXMOPt5VLLjebgwAW8= z=jXS&rMU#aaI0GlPA}B!5yfinY-W`b;dxH{n2~W24y$BI@U@`XezJ;w z@aI(X?^AD7Z*HZ}R>eZRQnsUFMiWFhIC4qXq|viMzcYQ=kbC}fNZ@4MynB7kaEZM8 zIGXi~E!pZ4?-5~i_t*^@Aztnvry#zJkqj+mIV##j;$T^rtw)*Cu&7Bd2k8Tv<3a)f(Gqae?;w+pc^+N1^ayLCq5`Tn<7N zC$8n0*Id?ag^42}rKy#wyriafRyx*d0k`PiA{L_FPego#iDNV4akDG( zqTP2kj>e7A-Ke>i(Pa%YG+ZriD+Y~)CEeKW7Al{>Zmgp-vZwRw!%V-D zCy0&Emp(*dhFCKSH%06vw2l85lV!IGkwxJbZE4n47h5S+zc>twY=3gg_S&hh@J-@) zoEOfO6-iOJ1zEKF?w^$MzhTRf?5UMo)mju3x1{8u64#=CE~pVw+YOT5ICSh2G>zD` zPs&dZoi9yOq~!cREdbxBWKeK)7)o;7y|$8>#Ylzw4&`ZUzE5XtH4vtgjE@@EabWaL6i+r=WU8eF}xzO>_kg zc>}k-L@yI;dlYa`!lVuQ#Syd81TW?Sn7jo|KH#ErZxswsS_j}gu29twvZF}L!!*pp zJ{4h%AyHsZs9##K{a1~4X1K5NE?rHgzJ(X(CIv0{6d5&L3R#-OMUi=` zu?>n*zE)Fl4b1$_$)@fzYtge7VrA63XaS9C)w`?sb82WZ7c)`A(v9X_r8#S-=IXh5 z{+lOXix=$V&xD;%_NSUk8mP+{ithaTe)@y%WJ5D>ux;!6fY8pTP~ThAePzx4YQ#N8 z%^VYz@ishIFvODzq}zVN;o(`I6xFL(|_i<`N%CGD%zN)2r>kC5K zjrZb=O@b_)4n3#FMt`oqxwrm;%mbChL|jFU*tMDZhMaYZ+=rx>HLV?j{zQUM4N|&{ z@-YePQnbVYuYDU;bVAKBAC-@uhhZ%ny6V(|sIu`BR9H6l=~{WaVt;m&zTM$Lr;sPr zR58Vnn>&5aZmQLna(48#%`Pa59Y5rUrmFk>d3$C<&b)+yBx>)Cy_*-6QN2$Ti9V{; zvaz58@faAlh@#~Pjw7Bn|-|oGtDxmJpfA|+8ma*f%+dgyn!OM)S)8JXUiBlh!)vkr_$x@#gZBjII zF}+8E6rLrDc0ALepJiO}$~m{4&$42fcoP2y@2{v`$%F|bPjony4`d6jtlw+-@OOc3 z?Mq$RlJeIM*XE4(TiA2AoL8yvFESLLpwd43g{_A{UFPw->Rq<;K*zGM-wUfzD!g}X zz81Xi(@NV}HR+f4$hWJ`_D*ZHc&nNvA!7=yF8HD1VLZ2hoya#HbmcotwHcD?Ve8zH9m=wx1j z$BZ7<1*e?U_O9h-Z}tXhOoqton8z*RhZ~Q{$(y5{xIuMP$J$9qzF zzb$tXl$m-UVb+qlSoElAD~@-IC}WKDhyR9%!8c_yw@tE4!?j|E9lN==hctO+$1J-$ zCSP7oxN$SiZ<)2Fd=7~<_Uh?cA?rp;7>)G37)LnXKwydZpb#!R%z4Ayhebfc1Nt3H z&<`9Q9&&@&5Fx>FKbfi5t#xRmzX@r-D;g=UD%l{S^E%hZKi422p`^Jv3(i7BHzw%v zW3bk4peIhrI*SEB2>ByIgGr$6i62=Pvwvnph*mOD9dC;*ZJ>2cUanicKh-oQ$|=^W z!HXuRO>BGdm{w3MhIZq2gA0`cgP&zD!_oL%*6Gto3&Q8+?(&l7-*sP*Q2(4Dv<>$; z;H=lWry%aP_<|w%9yYI>`ao??f^gkirDN;?bhu1apJ}ei{8lDI=}ogA5HCs`XRVcb z@U4Pa=M5Fh5^8R+&zhd8pfUZg3lCbu!hm?C=c{CO3xJF!v9!UGeP!2~e=;AtYxTM* zySwhrk^x@-6Q={7zk>&vl0TXA0{=?-ZWnbw_f-~21~>pqQHf<;|ILVmO2__dAClvW zST~~X$xdml5ccjaC@z5=RYNh(Bc6%=hBzJ{Y4*&nWgq+H#iz}zcc~H$N{T8mVg2tJ?E~>6 zIjWZjk3>;T-7L|bISq1IWX-`NlbmnScQ^aB7D0cdS(jnkk3O!;Ki4HMUGK>&5S#w4 z`|$UNm=P~CK1I}_ckR*NhrUkcA}O3^Uu-qQj`OeD&d)|3*lV`+E7DbqTV5lUop2@n zy1%H|mJpb__Bdo{gd)EsrN-law`to2{y)J4M!iM~tF^z+FTUB42&e2CR6V=l(t0BH zLsgIOG~^zB6w~#jr^Hu2XPO`12MD3qx_9=)6L&(60^$5w5LIFttobibL4he5W)MtnXMlE0BB2{ckIfZ2pU{%zHl^oIp z4iM^tIB-Ei_7gN>%8FBnTn9-?uu+lP_xptC8j)n%eLMve#G?#m-`fAuU=k%5j7j8Q zKW(9Z0rDJ;I>B*r!+m?kri)+x=JY{3FULOdx}AJ3GAKi<6=zI5_KY{0(5TmXO+?p6wuHWv~*1lhI6wMkF zMg0!4 z6~0RP` zBH3bB8~NoN$=u6Uza^e$3oSg*RJ5QQ!o@9bpPuuXqG8WW?!HuieY(4FmEhf?`r))A z`tkDvd4pZ=w+_P|*N^{y#@;b@qw<}lqEE_Ez*51@)jDBS$jtieEghcccZRz6z3xT}~@ zPraKT2dA@(8&|E9Tw7e57*llCbngu9J57S_w+BVb_FtNW_CBA~IXF1qxjkgRCt#PF zzMAsIYom{5MBwUNSXcDZV6Iu%1T*hhZOwi@w(|&Uor)mWO3w|&F)PkR9D}_kv7It| z@B3BvF}L2wveH>-v@Yb57QHOOczz)!xrkL2gt0=iu`4R>t9E}=hc=@&Do_X4_91Dz z<)UT%;;$nq5^v6lHO_*!VuMmKn^t-%L_4>d!5ZVSB8j zC4m_zEpmYK;z>fM7-~YHf9<{W@Tq|C{TWfp*=L02?jMSti+48}f5*FNS`K_URW{=B+ogvJcV4^$p!LAaE&s`uiKC3h$1lW(<(=%Uz+9yvay= ztkfVzOCnF96N(NoIXs@f^;*HTfrM5gjP$_s-oQWL!==^#U8UE4pjVXu!e!PNibviK zSpelzN=_Zy19Uog@1Qk|=K13~8G)hxE%PaGs>lw@Co{oYd6Z8$2Eeai0$jiHIrYu`k7(To7M?g8^xuKlf**h`=!r7KvRb^;u z#Gh)gKb4{J2s`(!RVWd`J|rLKVkb5-ilg<{&i&y4m)dCn)%wU_j~!j^&q8%`*UQ@5 zYJqBWM-TQym_H7$nBpdn;`lL&qVeP^%}EKVKTZys;9RPY_Q@ei(5~z*bW%>TW!j24 zsURtw!cF-$+Lt8l+#;8tC{e|z%rDSYy^Sj;FVsGV!r6P%swIIPYv0{f1=GZPhb1Wx zV?Dny+tA`4;}!Q25s&)gcC0my`}z2JSIk$Rcp|kX(RFPWnhkx9e$MB3PUOz%?G(8V z?fRSqN)CJfyY=zdLgr0%@RKW_M10W!a|9I{!7F_3SqvMDd zLvMY6(T^+qB2-uS*#iNI^)c+HkWZ4(l*N=L2N_)&iJ>Xep~ul1JpF# z3>;Y{m-@t*{2U@1Q=Yiy6XPl8Prb>Ow1k>YG0L8nw*-}cV9nA}Z}*`=BUGj|VU*RI z(;G5Y2+#B2zjBA^NeEF3-)(rG`|6b8gKYjM(!>2n9{(^>rD`j7G z1dV%1N|s_RY=vGvWG|dUaU6A0l1|+t)#DYbp`PdFdw?`cW5?`A^6rC;m^iW5T_z{E zh_x29(jR{m;xB;u0hTOygRg!5>4qfk+Qguol)U?*1XjJL5c2j1Aw7UA7Lt9XAk3x3 zdl$4YwY9Yw{EkGliIH*)(%^;y7cH=4AU+kz)op6Z1d}oZ7xzWd1w{Q%9ZNz+#u&o@ z0bcIw8bAXA$_q1PJa53c52R!SgYgV>sSrj4*2zrx*yid3(47Jj0?|ePJKTUrPObC2 zDCqgiD=MsZ=9)ks3s)oPyB`8sO*mHrly9kdkzgjSg^vfL%f{wrF~r+}?H7Q^J-)H* z1K?}F1POmE1YRJzZg9UBvFaEbD{E?Mj^B8pgrqFPw*hEG_-N3~fhE8aOfVoJT=8Jp zR0h*0Xhmh9W0RzW4w^r(jrEO;fXWUekBDs_q=>ND5z9Vkv5dg74`LI!N; z05M`1CUM`;kh;Zdh>n}s5Pvek4*a2q5AVPa9@357SI=g4WQMVbT+%*|@@&D`fa`8S zA}!SZ6UAk%>up39{f`f5RgHX}DMhO!%*9GwjDNIo@682xOjF54Vw*m2)L}x6lS}nE zlSs{3_CRm3A_p+p%E2kQkaNyUuYu)7?jO9t;oZN+=F>$4QdomVn3ud&i+P0w)Rf9u zl5jREh)IMm|GJMxxrB;qgFfP3t$C{VyCkk6IE_cjBJi3VMYg(CR{>=^ zeA{wqYLl%GuB%ZP(o^NjqIN3zet7yJ1pOm}J^#k@uM9(P0#2D5aM>+gJtSMaGP?R- zeMI?T{FmkGWc=%#-;Z$qPghhA%&i`-bx*l=hPc%SquM6s%jw76dQEG4+`BnVERk>Q zOXOrBbar&ivo{!9&bCjh>!CL;B7U72^EFCtx3UxgIzd>dbr93-dsrG%qy0K;znPMv z2ormiyHc)T2MH8{i$In>T?%-JG}&i9LU=b)>}`C#ERJ8;T?d>B!hPynUwWE>MAafHR}=(Y z2QWeqizlb}U4mjXH8u4uAUHtRkN(-a?aUlNT)hyjJOWY3siewCsN{bzOlx$e7z%8s3CmDskL2g78?~fP?5h?>XfPVshlQ zSX);o0xDY6?_j`aJO9b`U%dKivMPam5=5cF(~uOjlBWn?2S8PQVBBvz>E`+lw1I(i zt}=k@dsh@T?h@*tlNQU*&ka=$m$@G z1gi&)k^`WhAnZW8AovEcvWI$r4ht+Eu&5zk3YkIzaD%u6p3QE}HYS2uDG@vxxMdS? z(BO|qoUwzsbp_CBg6)=1NA>!%?q!xnmpVqqGeL{*!dFQH|DP28%>uhw?i`}=AiO={ z!z0;*!0%>y+fKK?)zb{FQd`uExTb1IyFH}IEPh?JSrGV{|MnzyRj!tShg`)b#&iS` zb>!$Dp3=0>lX~^T_HjNWW3JrE?6){vta!*DMLApXm~c@Qv{=~D3z~hxg;o*$iFYEtSS-rgcrL-OLyp1{eX3>pARaA4v zJf(uf?}W*P-%9Tm_cV-k#StVEix z>!w=4esi&Yb`Rs)hsO+g<8>eYEju*}Rxfkyy4-R4-QNCq?KtUX8F-^ZpfG-*GnRMykTkoj72m*iG@PO`in2hyTo*QoIQ9qeIx=l8lp z!n0anF#(DJnrVnp9|XXwt!F!(>j%(A`*W6}@c>Mcy(>!y+6mO;YMH#@;7bUbJD&HZD`MS8HW2*B1Hk@e!}VETCk#AT$8jbkWDQOMaAB+~O8z56 z652o!aSpW1GDtiHC+UM{+X5^aIeoC9KK!nHjbzf^Gpts_Mr8Sbn}Oki5rAwYPy01b zz<(_-PK&@MK}BB+9u}nNMPiWAr2)_iAEM;y>e{Rh+X7)KKp>WmwsyO@_k1_vQU!=W z4>+Ne4Gc*COKLO@9Rvg|2<{P+7}N$lwr?C?8K}%GDs9m}b0?s=v6z|Hf&)vzs5 zixC_Wv^5}oMEB-lF@BgV%I=n6 z5_OH+WKl?E=Hnk!R_5k(m_sEDBkhw!LLv_+v`Li8uX7Zl&r&kh|7D7O5P5@NaGRn) zva-%n8sI`^ziO> z%;-0)eKFP)ogK*6GCJeH{+(~Hg=rPo!Z;8M+oG`6vq9&^K(?gfiYzpE{vhN&9yofD zY(IdOO@rGVqI`=0#?sIrh$Q&JwE^HF^iXLK)&Q-hxTxrwl7cNlfdh;Q3MtnEH89qJ zD-B#xAgYF|0sd=%U6bSJn3=^*Y^Y&#p}?{Na3tW2AfpUyoGfY?K?}RE7HKB<_xwJ= zp0R19WA~Vc-ZVI#G?bMw;SWSXBJ%(YLZGjImyM0`9UYXQ`!8#rhl3gK58`$MjpiP> z2Qv^5J49or0Nj-b=QE)Rz0{!;V+|SV89!A`5el#QpZ})O#^Ud!8Qm8VNynp1y#-_^a*}&!Gebo zZX$4dh@Xq=B_kegUtnIG9^_vQ0-#A9zyQEtL;O@o?xE~IdzCPed0??a)%|>#Pa3QH zg;-H}DQ0relG-zV zLwjI>J4+Q2a~t!P#7xqg(ByP9>8m>n$DCX$1t{-dix{Uf%_Poz4dJN4kxm`Q*i!Th zFwW`ORr{}@Da+(MI-f6MmnuuB`2LR%f8oT}^iv~g7vA414&3SvTX6@8{{o1z`s3V$ z>UvsS#Y%1y-t{%c>J}0Fw)N{(t3ZUalnh_Rqxix;hKHI%qQA55u@I<}Gd6o-pw1no z~StkPbR$55LwnFTIbjx|CKQO(09U8GoqD z{bNd(R$?g&I{O*2HA3M8p#sN*?vCv=VQZ+c{`+!$0okTjjkB5=?ojJLd=m<2R|F+Q ztLet<2kR8_CGT9E>+7pe)hJ=!g!3H%#voi4u!<&Q60;z+7L&5|c!~1HZj3driT~Se z9r+$zfp73$kcD>zlFSF-_>#s(GJX9>xgqEy{FquCu{EaES-t;$evx525=#}zc6ikA z`tGkD9)r{FVV*Bbgq3aFs5+)xbM?ycnQ_Bo3ciQKNc8p6RNJKhpavAu?7$2){@y@N zyX&Xht7AhgEh*r1-!phq0KO0?w2+-Bad~2cgbQ~pJr!-X|Ko}wtwd(2dC>+Wzs>5F zgoTS8ZzQi6@%TNe`IH9c2EvhN2vieE$4!&~sFA^&MFfk6q>TU{jD!S-qdbkxC_r>7 zACQtYQ_pOdrDTb}Gzc!gWQbr2Mscd9>CIEBdp3qJ@fA)& zu?O3if~bVf+9o{8EMCP}2wI)PRIhrG(tW4uCMgpJvsT!z?+Jw4H#(&@o!g_1ZZ}=P zDG(44FvCJh>UfoAcMV>yg_5PjU+9zJ5$&T`QySqYQ*ObS)kp1@lyBF`LN^~OK{Pcz zE-d9EW#d%LyrX z?O+8B+|HZyFs%sLX1tZ+&C2leG@+QZSRzFUPsOQb@t^8dM{(eT{xA47_9kOEA4AVt zXbS^Y7L4lNryCpkzZ_U6y&Y>gL-IH|iZC{rt;R-94D&DZC@xJWxAw=B41>ZyI}2CY zWES^hJ07j4tfXpnJ~KX@v3Q#AiHaRFv@>^E+|}V~U7LR|?KW z002pHm)%0FY>3;;7r@uTU?D?la`4TQO62dpeRBWs00=J!KzD&kq#v;XBSku(y)-kP z=yvxW?=B>kg2@H^cwntgX2OjYSN?~u28#uPSxex|-==R}Nf7%F*9(cg-)rr#0iZ&q z-Ir{axwZ8e)-#~`o(c~lJUS$w8wo!J_+_D16eNKMS}*hs53eB*7|3-02*)DQ+D5Fa zKapnz^4DMX(0V}<1<&*zguMkt!myW;^163Hn zxgs0g`Gd#j61j)bZs^24k`{|Fmx-hKBQI}l`70w!<&CSD$p^;!chWCTzsJ5NAUR7C zwk-HW^^bwkWnvb8{(gVGnc|CTLp&0OrVHq=X~|YO_}^wSB#m%{bY-A(iWiP3#OmS? zfNNoP_5ZX04?)^MUaIUOH(-<~M}oe8At+~cnDCWy*ep?wfIj0Dy{clsqihzBqe>8p zF}`?%KTfGp;IwuVO`va&O^wT)5nn~%mOA;?99=P2CEd^LtWNtIL>Q3&lsQJkQZcU9VSu*dMs!YGnWO_IGCf-b|MHq5Mj@Q;1J}enjBM{<=0_ zv{y%@t*&S5tRrspC?PFDsmxOn8IvS36`7}^NHV1m zGF2oLp^Rn9ltg696s3?%NhpsDA)!b}DD&{G%iizzJO2AP_PgJ`D<1cKUFWdQb*?4K z7*cJFh=<%e)H4X!39HjgG&hOfNe6t`F#(jzvdjpM8TMv(SVkx-4=_GyPtZ_j}S%>_GPoan%!$Z(wvFm*@4t!2NxGN7arZE{-`#}i`$4Sd zZ6(ul(v#^I7Z;O1U+Oq(e^xaTXziLTo92)AO`eCV`pPZ;*y_;s$LbK1WM#!Gx{b*O zvOtDBlq@O8s6fgVYcaf>t63l8(eg5D3sFiB=Y-? zQl;V4w~se!9+mzO5PJIFz%R!0Wr92u4Kq)+<9WUQ+tj&b#@UwI%GbrtOmJQvJkeu4 zdZC(m;Q5b>|77gu(>;AvlHS~Hqd{ZM03Y@k_<$wjQhQ(}_wJ z3O<(?jn?E`khyw(^+ZZ2@8w!?k6&>KA9r1<7&#j%np@A-zYk2o|NhcuF3BlyA3b>* z5}JknB2aE8iLNec*RH|9tan({HGwKbX(@5Xb-@vnuy}Lh_U;JbQ0p%?>D8Glv|^+m zICend6NWLLmb|!Xf5EXQ#%B7yTwJO7r~ASIe5o>_6YONo}S`Q93V|o+jJ7 z*B~@Hfhc`xtaKVxYR7YDUPyZqH|V&*g@B4IqbJabJ4=?<61BN@V+Ybr4_1`$v2X6I z?RSRwAY3ujBT=zh4mKtVo0Z&!|2rhE6d?!!_{d2{6KEk(#DD_i$B&A~j~}07>mT?0 z8~`#}YikYKR2@{`_=~97y;_{=e4G{k1xT-}na-xLOjm0>HhfD9*7fM)va^NR=%}%6 zs%<_-V)D>rfec;NF$=HV$BwcGG12_PiR0Sn5=DrqQcWA4T?- zyLpR9oLwQXp}@4Gx7BX==6>s;(2hT~u5TC*J=`PJxu;+L*kGxJ(qkPBdQ-!0KDiXV zwjB@iKd3iucxLFO9haiB9X*Ux)Gzrv`ErwQH^pxX^l3^Qd0qSbeC|Ka*Pq89NHn=z zmN+3teOzULyG=cxQSg3)(X{5-rK#ze(I|z!X`h+!z57BY%O()CtA^% zwyEs>a-!#6HwR}xcf0AXY|xqIj=7?-yK%fPw=-OYUgP58Hk%|5-j`E_zQy71AmZq? z*NXeK-**p}8vf#xooiCF;WQPvfp1lMYi-%0_<0sH6{8Q{UW@NEgY7K;*%TXF$D9+b z{lDVX`3>pXAzrqX6OfsU+MW$S(4wG2l3>1lJ*U_2)x?YiV*G<9&z-mlfud)&(Zlnr zFLkWEcgejk-(`Z1JR~g#U}luCU;w>>7Xuv7k=l(*uk6DBxC<(u1XBROZ!r()S5r7= z;Jp)1|FS@MiW}Q^0AI?co;|hBrl(Sm;ART?4NNUsS5O6t0+^ywBD!7bMU;6Q zP`;otv4q+WfIEn5%Btvn!{l24kpU1RBnmG7%*?>Z0xHI!li-?=7~kam>yoRxJ487F zi2w&DKJoB@fbKP}sqX31+Nfo)cnyk`Gm5J|=r>~Kflk%P2nO}^$YGFXB+v;wm@flu zB{)f&nTJ_PDSnCEOJJ^IZa-TX$DUGNaH!LmL<4zBUZ9!S$Ghsjd)kGdTXg$F^X=5=nm+Ec|LGZfko|a$R{Hpk+ijZ%Ta@&^ zd`x_zz*#p`eQWb|yVRSVKbFt&yD0Jp(*(2Sa-_2;&)+8Xv>t03ziY{yLcB-xn+S1 zOEVX~$XTv2HQybS+zI%l%oX% z-a%Zrcu^D&`}ADNk%xNFiGiFG?|}mcpqin&&Fn_~_Z=>wkXRxjAkYHH>$#O0TQq5R z@YJ@ocpA%BasTj^jm zKQVqU+Su<|HG^UY%5lGR5a_ePAJ2rUVp>peH7C)v75&&t<6?6%8YzuR2b zdw6%5)A`?B9mvCRoR(p&t`(ym!v|__(VQKu10UIY`onYg@MPuK#@^zawsYw_@oQSu zt!UqoTh_OD^6J)5j&-5V$`*{&dG(m7tZ%#Nw9fL8hBwyPo}-(OE6lI`7&>fxt=E*M zz*Mj~=ybJZ^D~TEe%m&EOq+8G5i*g?vHCpos{h(Jd$ziSK*pWp%LO3J?0Wy;^0Qi( zgSBU0%3tuVD5+s#VL{y5cG}YyI;yA$j=-a<#M9m7*V$Yc3w`mPHgkMAT2xpmjiDmu zp(u^4$(Tih??6&!09SZaR3!lCg`m!Ked*1iTxyTL|AfLIGJ~mmhqZEWV&|wkoL-Xm z{z*!O)dj@jENY)44hAkPBaX`d*z~q3BR$=w-6I{?*GK_0xajBYb3G+vA$9Q!Qt_^` z**Y$N1_lP8;6of>U_7El`5+}?$#{t(#i$w)Cu!>fX|2>@*?eL80|5;-mT;8?>A_R& z{`!oOcdqi8MMU0kNa}2=Ae~($+=38JD{{w>@|+P=H``^T+o?rXA*qnWI8xBWas2tU z#}_MPMjxt6X+fWvJx-S=c*D(gg5F!y?6rdV(ydajR0wRWni^3V?Mj~FRd-Yf4L+|@ z?QVZ((_C-Xir z2iW;h1oV>zxjIESpHZnW-+bukZ^l?R{_c;hpx4DS&D%cJO86vAa7_J5an;ZR=?xqkDk+ zp{FD{@R>D8CE(jhg?mFke#{s;WDm=NSXmjYbjsK`!HW#HF&M9e+EWAP(r;>_cu9nX zAs7xF9J4l}cotv*@yP`Dx3X+>_|MQ8CgXX`D>pNq5Z0#6yu!7c8;tXViP2BMJI|oN zBShxA%_%=874LJovEfQkH}2V|C26*kpI3 z>2>^j#%6Zd6)RrWn;7(LG%^4$twOWvX!~ofk0--S_H9B`;Q4D`sCmZ1Ie|*_kVzOH zWsO4lsha-qBO+^f&9oFQ(?C()@-O^ zxR4|6Xm`2FIvs89EiN>y3f2PgHrEBi&eN1IhH5&FKJ%K+Jm5OaK|^zC`~}EczTQY{oEg8X8&S@&bCltXU+<5#}Q>S6SjVJOi4O{3q!&n$IVV zjlD_$Wy5qyh%mhIH*xu!g(4QQxP&%hl^88Ro1j!d<)6wM{H|B_zLwHLG1jvk+iuahj+u9+_wcSwwbr6&fNMc zv|&xEU^;vMZvD`4-j3suX|6Bni$}i}p71Z$V!zXG|3d2Zx`Ls*enZNX^X=RN#=cMAXE+JM(jG;|D&9zhKaI^I#q z$;rQtKK!I`q(A^M}YkPT|Xm7UAAN5GCIc)!Yi#fyV z;k)CqVWY;Wg1QNo88U-!xBgMZ{50CK~*Lb7Qx?pWSE_ev-VVq*=^*H zwRjVhwH5)1OnRr1dPNTvE1zxkq_1ARr=fM6o2yPbEo!Efk&k+ItHE1N@9EvQKY~4_ zd)7ic*yFBU5|j2)i&e*){5(R*vmI)FL`LpV%prpo;<&m5N`aoSG3cvuexMaAajWRD zu>{4sygtGcp*(PKJ&~TnoL%eS+N)Qu#z{NufQTGq$DwD64i!8f$SX%)6X+Oq_mc17 zvQ5Y81MmL*`xB2J17kEuMfREgyBF~Zfpsue)h?H6f3T6ZnLN2B8=<{VMGbkHSRnH-@Ma>LTrI(D;ut!$wgmLnAz0y?BW?KNu)?3mKM z`pEu%{)u-*nHz3Zj7IX=#X8VG+s4NmtFOJ>qI14Hb5e_4Bib6&CG7eWok|R)FcGJ? znXT`f`ry9#7pMPI%k!Og4HSl6%6QWG7awZIDIUPa#tIApUKPl^s5qPF-?eO`4%y`2 zd9P$3+%<>Z@x-xq=mevgFKYXG?O^kwRC^IV8%0X;^B&5g73-0bru30}Kh1HL$(|d> zu7d~fL-H8v?a52m;6Vf#RR}?*-=v1p@632V(T+y}tH!4US41RX`z` zc`AM>Cnv{BAyU=PopTMfN~ABxAKxVZCNcE~wMkk&r}j@J2}TKM@j#W+XL-prdC!Uk zYJtbtmp?CNz2xx&Ktw+M(Ib9`N)!-aZst2+qJqgGWYho%ifL_8kBpUZ=7~?=d?4_Q zn3|paorzl~1L8EFZEau}N>N**n0%WrNlx^dxnaf=pU02G?>`Ff zH5?8l`G zuM0?xM<*<^t|#?-XYs`hOkm4J-il|bzu=j3p- z?xjctCMc)7KPqA$lBFwhNOuTpE$Xki(+_8iM%FxWeRH<1ZJ~eSeIgyg2H1||w>=mP zzfwc{McS>wVEUwx1n*x>HHVmBzTF}s>`dxVU;#}J`389J>8IOb>Tj+%;-rOH7j%SM z6X;;!nvY9)K!8b)BF`ihBfqX%Z{Z8pd|vOK&DV}UYCIZkjX7@MhNloQKFkHH;A6?k zS!PH?&-N%ND6lW?lTor>m>E~D-di2|FvU;p@m5L3`VWqbw$58~M%b)gCIWpfWQb*I zg~DEQgizgOzfPe)OWSM74pS%<<;GN!N{!M_5N2LKydCevs1bW(eWAtBKdbEX)f}T; znj24j(u#lCcZ0g{k8LoY*n@vGugovK=3H%|S=5Lje!Fvt>H7qug9BrHl^>}8 zX}()y#n}Az^{UQdhgr>Ae9H<~JvdGu_2#&;a1aO4q);Q8-y~n8Big~ja%ZQobdWe} zpoo;s9<4PZTA!I6lfIq4k~}3Hvp43A!#RUUrORC0>QvmOwd%Kmd_Ar6D(1`T?puvM zX)0@QS!R$mj;5_y&olIjlVg@q_EKZqY{SKY_p_ywOJA~ns<$*e=42dB6-=?4*s6Ui zA{CoFIL1*#Y#pvMk``~)!fme|_|dqh*Kd!{+zrIy0eOG@6mOnX`2p^_t^UCZY%^rxcn7 zCpXXDhfk^1r9}4J1M_T`rt&$$sD(oeZMQsVj2e6)rk6Lk_V+es|A~qs+O;kLx94XT zm6rxwUIcUp^hP&iw;fnh*>$*2DA|v3gr}qVx8f-C+;BF7V8x*abQ!gqg8kZt7vKJs z-G5lPhU3K>mOiE`1A#-my;R#cGTc9|_P3zk8CjE(Qq_^*ep6}L*5eJ`5jVQxD%R(# zkLsmJHC23b{uOrU)A0c+bE8<(p999H5{kaGq^!XS%}kc_ zc^i6XzV4PXb-y>a@xc3}AB$BIzE$l!`_yl+-iJAp{F(xZBM)6=_bK_;Z5zKiv{JIQM;)erG+D6%3>BRV!QrMJsZvJ(ll6J-=mG#XEFPl=A zUbh?;P(6O_;$O@h->I zDPM+dPj_`GY;JLKN%*--_K+c;#fNR%ckt*e?COcw}|9Y@Ln?6&k*a|R=@Xw* zHbxrPRT2i3#ut4a2?Q}HZQxdMm(+^FhbPMe0y;@BYX{8>$?&M9rDggTRY=KwtBWGc zFDOGcS-oEB^7~+Nx%-;->Jj^284_9y`sTNnUiTh;Ftuq@WZFl6Gq3jB>uh*}88?SM z=h}FEU_3(7P5xKV-`Kjf!409`og!pCZ&@9q$qT9AQ{$;pHFr`xA*N~-%N3L*G;>JW zbV|o2{E&mO>&<4qfXt(*LNxp@=;T8pw@0tjGE=8Z3boF+eYN+_izx4w+n3BI42@}Y z4}Q_|5Nz;GXPqtiGL|tD??o@wy zE6*xr)#7aWzIkRkE?tF%Qf4F7?R6aog8hE}Ns&DiCw+I)k2Q3y`i+?lcHQsRt`(l6 z{;XBc!%R8Q5yw`<5tM#i%itFO9nFd{UxVCqp>_vl9hhu$(uz0G|5{fpyour`$(YD1 zGHwy*YPc@J{ho2Nib0R)Dru+d7v8Dr`0PuN1fQgq_m8R@^IQh=ee>amId3L|KAj$y z*=M8khw1iqDK~zZs{#dI)Z#^sDhUQBPplcNz0y?Z_>o<^V9_|%AW=Qi);gPdmCnZhpKsSW})LK>|aw;~jT{T{XVEU#Vq+OC_uHh6o~0Hfv8+WQ-sFT|Gp)81IZ zv{lJ88`JW+zrzRIM_387DkWtVjIszlZtDrt2$Q`4{Rop3PFHXwsseyd&~S}tVS>8N zR;7;_7;S~Wq#!f&iiiLyl@HZ+zDV`)is_&B4Os^cUXO0@wn$+=a%pF5I<#>)^xGHv zQQ!q9b#Ml5^HzYvDB2u ziuJ5`A=l*Y2sTs1@YAMP7N%n ztY7Tk9hJC?M{1g{VomZ<%WHSSGQTJ3NDV8krtxxT(Hyrp$s%$nlitserEn^}y-?w8 zSDm~NO`@mS=Rlj>G`El>wfHFZjdb9GhU|zPD^F5dc5ey2WMDqZ-}cbj@1&B~4^?S_ zl+fqTB8Q4@ur54mFf275G_28W5S(gJ*W5>Ku<3fL`YG>YgK2qNp0oFMo{f8Kv*Bvi z>Q9;plPSTEJaf~MH#yf4gg@jZGyDIz06mM{J$ax8(N>=M*^&u+Q0wy%Ucbj5>VoW8 zNw4r8GsJqJT{F!wWh!u^$z2yHdK2EY9AT7CpKihi(6N}F<}O5(XQlG9HGSvp9sPaF z;zmcsPKIr)&&Lu4jIVme${%x~F-JKV5fkIltH{(4TI#uG%CA4|>gRy9#}Gdgwi@V; z=IBwPsh&N!o}JUd7Z6F_VCtTBXFhh&}t*m^9+6A~P+@n@?;tzhYB$_`{&nrQ3xT5rfa1Rquq^;YIKdE9sr_@{Ovd zP}wW6^dbutjw`05NygfAe~~oYdN59RB=z-$%CWrO+`l?0k*y+1&i-elv~0y1G8_30 z^-9_H$bHLya(A2T<0o$)l(g@&cMoG2e4#SBociE)?rKSmHGE36>1RJru>9( zZnEA?7DAWqeF$CSwn8u1%0k0&#;~ojs(3Sm#J?iuvTBag-zaGC)U5(h8w;NN4mzN#{eZvy7LDcjoXTt#;em~s?@I_1c$|D6oQ+sZNBQr zc!2U$!1=6b<}XTY5G&&Np>k$hYxc(RXAghH{ummn7;=U8vei$K@rso+Hn{Ef)Ha6l zmdvzlUms>cy7s(8Nv@07{)*K;SuFoxI$zi5W*6H|?fFNgNfsv)Ms5}Kd)`p_oF;h2 zR`H*(y8f=@cIH!+*Ltasxv-m>4QBAQHC9l>OO%J+|2b9X>NEbgCWr5IOhD~}vy?f1 zz5okGtvHn}>+1^RwjVscHMf%)m!&28MW&#ux0OIJLnXUq>#f9TGZ&Nwt}2gDH-{ce z2%@6pkdw|E8g2jMZkMJLlq9An?5$}Za_G~oJzw_Gw3(_F-#hbp;*m0qcmSGk%;>~v zd85P`XxF1i;AhzIb$Y&_$N}3t>KPPl{)Bz@9oM*b>=EXP2s%319F6CgX>Ni{#0RS~ z1BG)f-dW}5fj?58hKPjViT`V1u#JOF*O2Yz_x4wpfAn%>8#38T+kcTW)H!3-#(O~L zAd%Xio)_Kym9K_lW8{W$?qe>a?-NH<_wUVSIKBLO==Ip*GVAf>l*Pl06Pe#R5(YJO z9tqJ{&uX)1k6V;3FLMr-uF~8@TmMFE?!sdkd!Z-|QIT*9lMl^N-%1}^D2XRFn^JGe zn89gYFr?zM-Q;QiK5I(<^p9^&ml6jt!(b=!VDftH8p!bs6V{U)cC}W*)!cWu zRaj)MzI*Bm@U#B&*HvC(rFXj>mq%#K!o$`h?CV zG5H?xkzoOSf$m=~lHqgd^=IBo2*}q@&k3@8*r{GHt9z^Ui&Xyvo9TgP<+^jZ^`}0xB!2sA zR@!p!>hr9<*M|+-S?*_r4?c?+{3~(EFUIOKUxL^zep*+`oBTN{W$l-KZI2>Q9o~!I z%@b0}xpOG_-Rm83dv>AP-T367!1c4W5i+;Ziyz*OIJsSkc9u_fOR_Aw6%8x=ES;o{YKOqB6_tiX2b=QFV>9+Z3juJWnfs zj(MFa-PRc8b>+<6Ta(#`7nfhN`3_H2lq=m*Oy2QhN<#BfSh`2b(CE1 z!pWw~ye_Pk*F-fQ3k`KYEtOWVOq?-qr`MzI5AEXDs#kF_@i8jVGHL7>kz?#NzM87w zZxx?BJ{w9e;+WJba^=mU$M?Ce9r0a-f;Doa$oV+^iNRpf;{1UhQxnq#>U@VM?1CCB%7Q#u*OOM;v;mG?voOnGgUFOLV{&@}&=-i5Gy}I;u|H zrk^>oBv&M?pJ~`?+d|8>CHh`>PYgit2(>dN$B*0U@kP~TCx`0N@+&fN zt1JgTvT#o0uR;CYT%!Kk^5L#*%Fd6HCVc)smCBc|9@{^(ap=zt%5(M|+Ar6tRmCo6 z`C7*b@{VYmg)EiGDQysE0;{0RseZH>U5T@Jn_IN1Y@z@&jaiFXBZ!p%D4%@%3r!eAywH3gYz0cX{ z*koQ;1#LmH8I~-l5h8E;2W|T0F|}Ap#>GW?9kn4U<%(^4YcueEBgbX#aM>k!a8tKJIVGtPgBXmJ~I$w*?UW3;<*=8d<<~9x@ zp(xjhj?RA$7I__e?a#?E1p%$&rMTX)zzyT69}1{`KoOVq8`9u7^H*ly#>iXQViFn8 z=4O1g@Nx0h^B-p&`J1+%QgLTH!0U8}?=a8G$72x;cB$~#Pk)pm z#jL8RcQ>1T!#W9%cTck7pN)!%J>KK3I0tw?ZuOGAW1#P3|KXihlg`}hqfnBXXd78f zsfh0D?}~Bfroa15HrI~HXq)Bzn>T}kBzQ79BM<-Eeiu=suss*|*F)|gB5?8!EYl`v z+qTlLy$~PJIoddP$-mpNw%6(Hi;AK5pNDFyX4S)0ZrPV#KfYCHlUeeIb+LsRZ<(Uk z@{*6sUG)}psoE_%aut7uCc(-PrH3ce0>e{u#KvEaBj{3mF`KWqLC4nND7v^xK_$_ zr48eG#=O#MJQRnGE63pQ`eiPI5al_)KkMuQ=|eYWR&Wol7ndW){{{o6|IPn?yp?ZV zL-XH%4z#}09?z#Qq)L7h&r2w0nVWOgSuvqgguHodng5r3VO-0P=a^R|HwCs0v%BBt zjo5$~VPphfHOr##V!+0K|Mrf1{`Qpf{}sDmD+;xlFiT~xQ+Q0wew$$l1VaRvXzSj; zSExtFTxGTJ4Y0j{KQO;&IUtiK90Cq-hr1R9|Bn30U0>7hjMZtXKRT2_mlsFe0!RpB z3WBpJ_FgKBw6GJ-O!GjH29ct2ugBEomhUbR_wVo7E@N`okB_{*qq@3!x{}+D^GZim zN%WYTuTzb1bskiPc%Ms0Tg@1pdW~OQWsHh!6y!QSmcoP4*3RGFGA=EREhiDPEm7|w zjX3X%CM+-AM+vhHTpvUx!h|Oa6yUj7rf;UL{EFj6wx5vI@T+8hj<9Wr+Xr@S5i$;~ zLd8smM=@sZ9gpc*O`=udXY;glY6pN7(0?>3R((}LrXJmH*l@CqcEH6ZzK|K(iD(m% z6)QIH?>c<>^6cYHtaz zFnF}Hm=I+3Gnql-oShsjI8)(gK_zr{B%F~M53-N9eIGU5{mI2mVp}Lutg)UK(iV)P zgK~BnM}I0Wi#KpjSi!_th|@4}k$(d$Vzy^_-bd2<9W$X)V>BGFZ@oci82iDR>GEZ7 ztiysPnvQi^thfY}!>8=gg9qFNrWnlV$c?WB!F8e9d|{nA)(AYo8k(t~zjj!RmCe8b zNoyABaPKZ&o7{Qj%9Zt_(^S-SzsZgxl0n&I;+WIo-?mAz2W`_3FHTWt9+D!P$p?F8 z>5hi7^}>!|w4}uNAjfI6mz7JBp38%W&u++&!{1 z*#o(b7endy$qFZeCk)fAqUXbwXXw9N9%Lt+Lt@`aEWpc_7sn`VZTp2VV_BS^#V3D_ zRfS`~SMoK%fm+6qX#_8YtVdRpc_CTLruUhFvcQH9$#&0hAUVAM=m}`WS1m6jFGqnB z_WSAb+*3lm{ZX=)Q_3J=rkaK1}hFw#^W*NeJb*~d~Q%4=zNYPOMIJDzZ=PPWh!LvkQ>96_ny;@G6 z1Omeo#;1KxSt3joS1oqzBNn*j^W9^{C9b>SkB3q=ln^4pGY)`Tc37NmldEh4ytwSn zor6xn#SfMcWaeD_ag=z-*Rp%yGQ%R*7U@O~+b3)X-R)a0mfOV>HX7MrAMdZ7#PgG| zes@>Vgb{QyxxXz2WD6WNlUN=kmk92s#6SI^{JB3RkR%E*Q! z>w&Oo!a{b?2^kiv+IujJ1TNx?G)^uV)>ohD6?elrk}$xL+Z@~Nx$5&*me2z>?|hDj zynFB7ba=Ikdi8F?N_hEKz-?>isCFxWPzW9xI7qBd+v;HeLa1(JGX>1N2@A;p$b-NUkL((=%rFb z#vBLTN6vt36Sv!9{rWfbC}(`BsB2oLPDZO@mkW$0b_Bo2S??hCFw?KFWJz0@0%mD_ zJ}9%V=6*iG2pm!pdPZBZnEuZ{)kCgC7DGR?CH7nxR%;5bPCF8^7x&*?be!_+0$rq@?GZu>x!0dxAIA$5o*fk_{cZtO^*Q9AUdAO);WCAOQagZrIL-j?1&>GZSRp zf+-m~28M<=knRf`utG~j3JCpctVc(VBO@)1&!GOOkr6f}Fwek`Y?R)$qcY9uCC;z& zYOfr*0i*82`Z^!H(tTZz?Me(z>5ww~OK&jpQ9{JE@=QvoSKd>|7)R@W6NzCL_%|q_ ztHT5ytq9bTB&cp;yLjAlLtrIF0e4=Eawpc z^F@z@fV)bBYXJFyzddZe91n9(EHRM%4G#Arh~6+$a}S((MC(8(J|S0h5rc0o8g z?cyC4TphVTACyTmAc}NH9u~;6Wzc0J@e2GxR$6R_kpkWQ<)O(g-=z<&tw%I8>T@fK zK4#%#1JzGck6OP@-_WSKfBZvpGf|MK4dbjGa)sFI`a?ZKu7G!t1S8cY0?@FK$Lt&I zwbk^{?D6z_r+qtnRgI)HyH#|yrz!Mv9BZP|SPtT1`d z+%@T_zxbFG^_aUHr&BJVq7>JPa+N(aKs}8b4H2^+Tb;Hwx3?Qdt3rRa&5ez&Hhi^( zs-BpZq`~_uQWCWOL}><^J(2tx>$~}Viah1Ch3>ui?y~tOCsla{Wk@ekQ9OLNB*X(C zCDcHc2A4*fmTu?d>|LpjHt&=^V*)BFtC5kU$(q5bKr4v@Sz)V>?naq2~4a`^3E3vDnh& zxXnwE`W-FO0TyRiYjUYlAF5+99GeHd=%i{!-I4W=%COZ)X{CNVC!hiLkD(EJ%lc?* z>+?G8_C=H0Mp8QvP|(CapZa{^1*$(3&JtR#RCd3Fv8Kea8Ajv{-A&OLX>ud z5)iPSkaan({NN@A^;q|rxnDlJlhg==R-SG**l-tUs(<4y%j(0+oH?bacyQ_px(A0j z)9q{|j#ahuDMsdb`lh{p&3)k{3)44drCU0ose+zt@HEaGKXLY1JxLK1 ziqCo=r0-8s8S5osF-^%!aSsG-y*sw_c{>&#lx#RI$oxi1OG>LUUlL7t6sQ`Uyz^NeApvn>a#q40)2zf7 zmj?U(UI8M+!$ybNR}rrw&v?Y2l5K50cSKdS`f=jqSqp<>!Tx* zlbcZZG!!(+*W(iYzETQ2VGCU#wKYA;ZJvnNgOk?5(P3Muo~bpCE>p!>fn@~J6}t5g zZr-}ftTN`fDdfx32?W#cQj8RT>qyD%oYVO^o~C_lf0`bOa!+L~deeD9@k^#PCskr4zKHhgGlxnaQ?$EA)s|5~|^=BI5korgcpL72b= ze#R^1Pz4u4Hf9#!+#;ZX3Uz&{;v3SenHl@umod0=g2W_IYjKOb71iErveg2a6JuQ`^`$LALpKAA6W#Ttx#d#*1;8}O-!AP&Nb zfBLqN>a(B~VIigwIjXkh^4W=Rbx=@ofnb;7%{Ewd5W816Zi2~$Y_BSGFBQu|_4yJe z=TJ(e>F-9h5ZWN1JY;!qqytaaVE22skF7f&?Z3;#rlFTG7!k07C!qq?s}j@vd^MFd zT{WjIsEv2O?X6z4ULm^;(sM8N$m1S>q=uptl@vDQ`hxf5uwLr4j~77h>xXTp2-Zi* zs&Q;O)|cS<5)~EY-!n!yiKO;~B!%v&Q=1M0jD+AeDFz=(%V5Fb1x!bw3NW5xS4haO z#B>uzaL<>Q=Ia_79zA}13r@-;-6BNxkDp;(51ulBph$E8R}80bLdpz3T_gGeEL?fY zEn1$*G!!z(cnEu4^kPN0`fHJ4vEEX zzA#D-6RZ#sziQOWs9F$H4_x^ph^7`RCkxk>AOE5s)IJ!C^A25!1_Tb26J+HCwFSgh zbdByoj1A_W*tKE-4mWHR!dNq(ot^E1Ob)U(R?A>cPY4FTzbX=+ zdGrhivgY9t{FQvV01KjW6 z3HzMcmQpUQ2XEtN&z_BZDZP&yAY6|jJ2)-%knYidb8CEhp4Y4kDXyWIa24NF8PzaQ z2xzn>LJa73N#?9@y;7SSCWb?<@t$(>=Y+(Kj9#E4!968qb6j>DbgNvTZbc}ZpfRVV z-B4JhD11jbOg(AuefadL8h5Q74Kq&t zcThhd)8h|W77jp5cqCZHC)F~F9~=dMDuX1@V@c%wohYKvccE=B%U;vBKqx*+ZAKsS z`0-&qW`H84nunqYz*@eHSN;0&7aROqR@R$*VdByZiV1LEHaSo7iHHmsDym*5F%%;h zGNMdQgK7~Zy223niQl`PPXbN|VU|Iw1~Xo>HnO6KutSzfFs?z>1^0Al?75G9(E^7D z)p%iHp%W4`Hmdgj3=P6lkKjG(IoYruS~9fggW(Abm`gH_6WNXkuAzMgG$0WE=QR~Q z;^&=I)eG{w{>KFngzi5!5iugh(I>mSplc8{kFcd5VMpA$Ih0&e96C3(cNI+Jt<+GyXbWAzI&p;4jsv|xAom^S{6z}Z{yc| zKh-nGqI2jkGp+#9o+4*Be{$)A_pUfAtpCPkKEk{tx3l((lmk8mII(@>Gr3_gckj}^ z7@o(Y$H}4(tgHZh15a(%C~sg>&V5xI5qOY=*`yBX$|err#j+i`au@GpSGYR+ZNb@jN>lB>fJ_kn5#YChPfd%+SZBhE54 zH`mFqsM2ZBA1qB|+V!6pP^Cgpf-HL?Cgbd3HJk48A-pead_W%u(0~Enh(OcBvtk<~ zZ|g;`b2!9!+)=LPBgMO2>Rx=XX->j{q<2gT()WYE-TgOC0p_)cWr;8jWU4E~f_T@4P}GYsrKIX(dmQrZ`M5c+I2P1Hy3zigKBhuB`)V2ApQz&gn+q zCEb)(zmVS*pT#5T^uHU$8ESWuPhVDx-C~M;EcJz?CXe-3=Z@ztH8KNQBJ9qqnMjaP=hNcQd8m(aH_#-~6qOBt| zciu?d9cf2pwm>C>nuHqp+X-b=W2Ni+nK`o1bd6<;+WOX&UMy(wZ}ZR&lc=@NcC z*u%OjsU>KaTS=qZ&i{Q%wO;3rSULHaJ?Zzv{>fUEVHR~{C6mrDtB3P#5)6Lx+>b3< z^zqr&Z#>!_0d_;7AG`$=<>HQHHUIDVasCc@kD46WhPvo0fd< z4hJFk`!sJ~75$e{R?-T0evFV7*MEPO{KAYd3N+conz`EpzRTG?W0kG_^s`>vRl1St z|9fKiZ8dTfDH2=|CQ(ZO(}hYO^6S@EO<3W8Z}JBU#h)_7hsGEf#>iCOzJr0*CIG1@ z*fy5qKn_Fxz&?FJhaiMG3k#WxJMUy=#k}IwPLQbpW~3y{(ogOfN7$f2&-BIeMQPE} zKTJSmw}r5X%)OsKjY1BM+xz%^o638dK0Z9!QR21-jAEjyGchs2-GmN=7Zrm z{!$%)6sL+EzsK1oSztjZX5mUDuAT}{L|q9oG#~&X6>#4Buk&=Yo~n{-H6`R z{2~eF1T4hx8%$8oKOc|UIXJL4FVk_zy#-S6QA&g|2J8;SGSN!_Oq)==OvAl5VFkYY z-;s*V5Trq@BA7OExptCV-BX{%3s`XoG?R}|%XRKQLOuWn2n3qnBA|JgQYhXbO;%B% z<#3x*Aqk}tW^WuEUcVKW#yOWGM15xK$@aS3#9LnEh2N21o8Noij)jBK__ji%ivK@s@88q=mSzRsAh9u7AsKBF#tZQl zV>K6{rGqz(Y64Mc;vp0F{5h)@MFxo)y%WaSCMF%V zH2T0i30g-nPHe~*1`M@JC+S;zo+XF^q<$cPQ9UYTyk8mD9}E~WlMU@*SpWp*tMMSE zv>pQbgEtNk8;E&euUS=f-RYvn1Ij}s4i5TKXi>PX)EIr={pmo~3e6RtnomSZ;3{W1pB(6LwZbz{>7 zkaS+^i>H}h2ql*lbI;_A;`S}DN)13#KcDEAJ}W4&U)ie++72X zsnoYwkP0B5hegqb=g$4@5F^o%pXDPs4$(>@coN(O)_=WVKEHzqo5vt@h{a*^!K^1an0aj>^0y?6 zpqxy?TPEwucy>fXfr!20NG|{u=LIl_!8j~5m!~c+2VmR>`G=`>)an?$eR!bEChrxC z+72*P;^p`G!qB!d!7v9~PY`QGe8x>#c)I+z9I_46XVAC2T9~LuTjl%i-7%uUl8@j= zM5(P!&>Q~7O0f(27NSt=_`G*LDx2BaCZ9&g#J-Kvk*`BQ$rTRaJsO(p^8FaircKx% zlUrVxB*KKnKJ&#lDlNWvO=>}Hv;C!qXgM|`o)5%d&}F_!?yYhl-8j}RZT7h6erL(p z;&`@k1zK+hNRE0=)a?S?a0aarZY0oFZJj zsuF2zHW{Z?D^`d7JxilKt7f7UX16%L_|8HwG6*Bg4*D$FhO!?#0(9eqsq;D_r-;~q zXySrGo{a2C!XWyi5c#$d+O;y*28)J#34}+5LI>gcK12jEVZf~ZCgjRqk{5U9qL-y^ z4JiP2Da>xfq9s-rSNPZDpIC;QfNmPF6WrSVmpzMnA<))^?uDTFUbFS)jha{YG2~`J z66DdNo9Cp0v$CLyDuHHJJ6RqR?saqiCx{9k;6}3MNB$;^lojP?G%%WpviA=DdhYMs zUqiZk0yPt%N=ylfv2tcuE zUz=$q;zK479w>Xr0jznFQxg2I=0VNBa_R~4-a9B#ml1$SuDnIOaXvRfN#n-IH(Iz3Um-&CIu$c`K8 z4@5HuTG&v%ufx{{{E!ZjvR4&BYJ6|ds7qF47KotZ$H4tQLIlD2Nzg+H*a4+&+$}N@ zg`Ntww<<~WL;a-zL#9ufx#tdJM-#OjX%>xz1*o4fY*SC~Wct4v-vrSAEu@A52vV$Q zZD0H%N2ckxgqi>R!#?$E5%njLAVOxra${WT=PBkotz8*k+d6HEXdB+3Da`hUlqItXUJdn-}fzQ4M5jW74=0CG|Q$r1Si z5>S!J$$oMCg`Dluzc1}YeF(X99i055VH^a`07`K-ejw$rG8D7!+_}?rW%!@Ccp;P? zVD&>R6vaF+q!hay_?hEIfx*Po@3Wsz(!mH5-xkBV$7Qud))@3eT_d;llblqIbhL5l zb%6Y2unYtESJ&bXkwG>!eUH`|-2TT|2oqpNzjr|a>HZQ8Ol7MR;7=K(f*)Af zPd`50xx+e!{B6C|ibICd`vv9;c(z~PM)1nbev1N@vsv@L^c@V^X(>`9Fsv^Pr3?|< z0asZO>O#Di6&}rhah+$6U3?8F5oBJE<|+GBaVRM?P(*;8GW7H3j)?@!fY68mXu%y& zjN6KNNW4J09o@k*K4ppvzU9Lu+I&nEVzj7#qOC#P!PHb&7iYWaRc&DDX<%=tOa{tv zO&T$^Gs(BwhyW({`!&_Jmc1*LG9jK36*YQ=1+kv_FVnHEiNUycQK z+D2P@|BzVuA1d#kc*`08BCt@7fX+Q4G{2--tX({bzC=pX-)|q4g3_&!UH>Cqt@QKR zE>kIP00D8QhzL~G799vs#DXr7ayzIXdCP{i7#!MvE(~gHY<%AjHMOp;_juKNuau%G zMf?Jv2jf)*!Gnj}+|Y9(Idhu0c7s!f5h8$Y1SVBkSPMUKl&{b|j{_5Mh7Tg=)-&%Ff7h>ka~60;OaxLdJs2!vrw_qM67<67^h ziJlA8R(dnh`g~CDB{OuA2N2ePwgKh-mx0v(_X?mpOe;>E>P)eFhMCX3V_4>otutm) z^78t3ME~CzGqL2N^pDuzzJIYS4ka1s`+z=eSGcy2V%?O)l7%V|Wp6Hs3rSuR>R>Hs zO6{`wfqJsm>1pF|`&M55+pv-ZK*Hrb;@uu!5qQ{AYgA ze@a;*4zj?TKL=uyA6TQ)?DrTLczi=!$5ZEe>i>c<&mPtQGDAZSCjlE@7V1-ic#az8 zb4)~Gtv*^rS&!M(ECVeG0xL>t^fQ|AQX3KcW6Bq-iOIu?KO~9=Xio*FIw9erM+U_x z6@LU1D}+tNllq066#<#JuZPjrgL{Ui4hTGI&pvXru?WaU#JFHesQ>{ZTqh^chr}SP z?P}I!WrINt<;xeOvY`}*XPu)2i^glx2LTAYw?G?J{1EsyIGi;$1pW!02&3I8Vp2Q1O4Pp6+Hvobx z$S{CkoKQ?bq}l4#X;G45E6y-fieZTbZS!$dyLc{$oh#sr!cxD&4Civcj~Iw>u4cBt zLP^>H4va^H8S~@)S(3p4GraXpvRug3p5mjtA_Upu$0~eobYj!rJ>IDFk$e(GVW@ zm8(||c+H4`#J`O>_w_mueFvTv}@6mJhn$KE&JzYg1-7ih$ z{~vX4{S{ZxWq~Fk!6gX}0TKxA?hxD|cyPDi5ZprW;O;c;(6~bf4~@IK(>OFP@8+8~ zvu4)&3*L|PYO3qjJymti-n({rueEdUIt%SO&)j>D7y&d0Kq|3XjC>doyw-h%fB+;c z1;8ExDAkQkOn`{%>F@W0$zuXQ0U-JTqDz2~knkQO?$AS(l9?H+U0=!7l?ySz=m$MD z85tQlIXMBa^zDi40A8-*5`g_xs*I#0BA`+Om?a=z=>A9G0V(r13?3L52;9dPIbfPw zUXC8#+XbweXK=U~ApOqpz1#2U?R`f?B&()|18kST1_Kz`i+g(V0Sy5t7w8`%K~4@- zJ=CnMu>fuc>_b4*?bZx3QE+kLMM{b}IIwhfb^>tYKlLHtsRi&Xxqy4l)|M&ogea(} zfG!&sodG*>ZYSe(?OcDKnf?dip-iS zPk@e?pwhH_JYLJlL4@(|EiG0GJ{L~zkA1KAdc??7#WP+KZSu5nNN?prk;d~%7oj_@KfciAaiwcsrnH!{!JCj~HdsEwl9qc{&b&9c+Z;W@Dw#D3?B8 zCT9FV;NlS=I*BgS0nbK1tz=m*#FGYeR?VL9W{!+AHePwVjdIzn4u#kgk&^zi)dqHf z|11DwV`J^mjkUFpzz|$2%P_A;Jd2Cn80YN1Wr&3SLw+Zr;qMvq!E7&&MS6bgkaPJJx z;q6@WoNjMw#9C74GTry?$vRb+4|1=qsCk{c~Cb=r$n6IkXY$w2$;f z;vW)}qaESkk9AQ^gGC=bQl!;b$b~}KMzz{go9ycKRAbtK{W&r*VuS?u49mC;O>Y4(~SgG z#ZPo}vTT~6V*jOO|L-0?U;%;k4=x5O*&{b=hO@~@OOBqNp2O?IScG?Yc#_i6p=~6e zdp>;zP7(e0KOuj5lJZ`?{kU<;i{pAGMgE+w_OxhV$R z?rj?SZ$P4CiQpzmX#Z20^{@g$G+;4(2~3qecnzlNzv(4K_P^Tsuf|gH6B+(zK>yQ+ zo@(j;T}k}=8V}WfDP#ZF7yCc|f1CQhj{!9HA^!h>2EZqwCUO6pI{5d4j`IHp#{AF2 z$o!u~6X-$@>;EZl{{P;o{N(>L=-%mF+f_!jyl1cY+EEOI4K9&t4i`3!=f3xDEB{0U zTs#a6D2TvFAua^1i)cSJP68khluSwIF)Mfz$+wUrcbq+Rq1QL5V;?S0NS513sWrke z8dWSYI$qxpj+5qjeCoNr9T#~ISlT+YD2IIJCAyTPKwOqqfVp5Q8`K3U1d)iF4R7Dj z4%`rLZ6`k?Mh`}L7``Z*{UWZ$(+6rAQG+wdA?w6Hw(Oarsoe!Mq zzwo06gC>pg8Vlm|A{Cu1STx-W;T&vWm?dKy_P2xTx#t1h!kBBU2{Y{P*#5g)9LzeEkKNb&&&RTexoVb zuyE!)6iynVuD3i-UC@xvo1c?AwEjn(D#>d240D(fKws z9q6JH3pXlkmZrrGq3C!+vJ`s+>7cLrSa!*1MJ6f7eW<89978GWi76jJ?F($+NqMm| z4KCce_gSpT9;*HCfVH>al;N}%(CFIYSrdF%~@wpMF#Zis-0fQ`Ibdu)A;-M!tn;rx2|}Uh@`rN zKsepZ*9zPmen+hHH3dED9JkwGz7%RPIq9Iq@`2Oe_Md?AEXPEKGUz?l`f8=oFJjqz zvNtuZ|2~AfRK-K%zJGYPnJPpXWUpvO8=ZMonpwW88)dYFjyEDpJ#-CGzQVa$;o|}? zipJANq+aaAO;s$Lpr8Pg49q^jQ0xAXzCoiTd&@W}B|-9e?^< z{W126yBER}TII%dV}?Y=agjoZPgNv%#W%J-Q7|PWxQbWM3u%Qz+xsB^86BAlmLSJt zVMioV*xq7<-hgwlu4At}#n&?ygF9y(AVI zM`E+klzQY5q{i)r?R9P|keVbD`26e|BEfq%f4084 zvjmuy6Tsc}jE`$IV#3FCH3vo*VOp)O2>icG_rvcA!2+3*HYuy?duc{w2YD4GNgEHg zz+tDLqgzT2IG=R;n*9fE+Yf#AYO0k{whXtM%WW+Uuav$~L0V48mRvj0e?`q444D+Q zRwVWV5-!j0>zh#rppI3UvK_n2V+kf6D$FVZ@Q6j>j+@k?a^arw1m zAZw;1|!cse9~qd6E%JMB_p6f*Eu0MFkvp z%YL!v4xT=z|5bALl)%pWOpB`bgVw*5j%{vI(np<^zrn`%Vul1KX_jqGVK<9=GKJt> zPpn|;W&y;8^ZDkH%-v54$#avE(5^Y%;L)3|6VXA-i&pFY6S%E4SbsaftvIafms3kHZl|Nw!$av} z^W%YP*hOOJc+Pr$ZAU1)tRs8C>T|D9uv`X(eW15J{^UaM3>Iix=~{TCZ$ac$-^kiI zXw+<`Cp4B1YWkVxt8%*$?zBdby(t3g#fb6oMn-^koAr_P+90>V&LggwWw&#RuPqN^ z-Z4SDz5bzCq>auaCGp*11lqTs)7yy~-Y-4`kZ^F#sKzx0DWo^XXns{xjrPa3YXQ}j z<6LDy&)LS>BB_r%67jh?O@2c+xUqzW?UfXIcfWG~%_0@@+j(v}X~wOa4@mLqJ@MG9 z9F#^8gmOiwU=p1R?TkVmvCCjb;su}~vLpd$FYmh$N?u{UYx#8G#RWlXCMQ*( zmAR5F6~8~#$3OfgWlASyL@N0Zr&tPHJ^0<#?k>)4#XTNZ2fELN=B^RN96+=2lEC9F3@9a^21XVC=Rb-Wj2 z#|s1aWQ>~^!<;G0Z~5#HEwV103r8~gnzjg3L_h6al&o!=Ho zg$&qq`=Hm2>%-ng7Vy(p9ds4`NH|#%aRFO%NN-eMDOa7l=-=o5EG!jmz@KI{bk2?N z{ibJtML^dwICBO~2dS!PDJXX`9q4-i5dTCU)44jFL|8A|diTJh}- zP+Fk}gmp^Pdo(`mBPLc`Wo`!B;UooW$@t1&XF_|oDNS|JF&h&W1L!pL=?e@509>a% z;;cuKVgN6x-O%(r!egoFqpU{jZy$k_ZjNajRh1RdXoN#b zkAxYxv-bDwh>+JbNQ{&Qz_l~cZ8-#Wq)Wh+~WSlnOh ztdS((dUF}1uYNH$L7KnLSQiylL_BmMwlL}=3f^w1|8Tr5C%6(+uW(4y(K-9j|! zdH16r=Zm#PMtk$UwSLXXl!IKVHR}{lBq>9O83P}Z*FHaQ7B$0|2c?_#u$H`zO2oYu zZ*EeNnx0FrO6KdMWudR;nVX8L8B%eQ8A!<$8V)#gf7GyY{A?TyU>5XRo63WW2M`bCA;1dRJoNN>3ERn3>D zcrBJNtc*~D1%}IDAr4Kt6mJw@at}R(AN+KQNN(GEcA1RrTr)TvLjE{N@e9-urc`D0 zO!;SzOk`$hav;Np+?l<5ziESM=0~98or~K=vb7RbpHkEsFGC$7ITh1JgC9eWNy%pO zs$MV1U{w^fO8|ZK{FQ3>+n)%OL>#O;cArl2iM;2F-(@Spahpsnne+IbmJ4^xzV$ak zGhBSv)sZ>DDJ2%?_ZjStu1BT=ncNp+>5GJMDpXPAL?&o9Y=5M z^+@ZP(8vXPHM}CE>9#UtewuO{E`_c~V7K%KEn2Ymu0#BZ%@4EW%!q(9C&Lj=a(Rt$ z8l60o8y!VWC%!`d?%sz*W*}?0JYyPNeNiTz7qm2mj1m*NtPsmQFgdz|l}i1!^$6Fl zWkY|4i$8j9i%4{Xce~V!*Qz@4oH^NHdKxRvUN|lH)YDkiIfb1fJZ;@ml+B#gV&U~e zR4kIpCiXzNvpv;)Px#!!grKg=D*w04d&04O?@+k`vy}{)y5l`^^9Ay#(Wq6w&-x(>ujgFTAnn(IB8$?Fr0`Y%aD=OrJ3n2M_Uqxx_LyC(7C z-s93-gl??uIHC*5fBV7vY~Q*4?LgCEcDnKWps?kUnPGMh8L$`b{AB~}DDH84c%_)Z zib?jEV^6~&WWsYC*euWVyqG$8{WwwX`?+l*5bdl%EWUz#xI3f}ell5+1 zMd0v6`WK^wH(gZY^MaRU4If^!sfgt}@xKmmP45l1zkIy|N*}Z;8Z{qSzOJ^BBe|j| zxF?K}KA1E7p^f2=!)F3G?@_Mh-5a?XVa3+^TXdt@lLN zW(kb4fNoXaq&G|gs0B}QbHGHGv2@00LJ!DykRXyZ^imbP85ZdAgIB&sY3L*8jkTFd zo=6e*3q)fog-@JN5M1HW{aUL7PVhqP@*7oA6vf}BfKR*1UoYItr|jkxHHTh1_4PV$ zwcS<{dgT`_Uzq=H-6VS*+0wtxQ&9UO4`1p~#Ak2e zik@>lTXj>s^t*kyj77i15fV^jSJZ{`GH zn0f9n4|N&%{TSt=1!v}tuI$8T+U~ut4MoNSiS zeALf^3u|QKVT#Y_p}8UszvQzlg%%Yu|JsaBcTltO+jO~9)&2SlRq+!$UemInC#wqP~~OdHZbD`oj}Q^kt-nfnoXEl0?y(Ilf=Nclq?$RME`1IQGE~ zrabM3BEuV%{#28os(p7BCQaotK@gQ%nI{=AV)*$ZM_OHmj5yJcy5s9^MDjl7wJ1tv z2aU3zKTRm6`9VjG70xOA^AF=XHdb}-GfBjXqphwj>c#V!wxXUJ6F{7p?^5GBdmOma z_5I>U!&^>MKPb9?T|1Ux#PD^TXXt>vE6iMsVsa9g-$aPIy_&X#k^8v_`8wmrygOIT z6w7hIa;rJ&E?By5EU+mh{_JkUl>B(i{BRZLA&%sIUow8z%jWfjlFzIor#bvqYm_; zEN>)ZdHD~Jrb2{n9%8QjIXxo~5nA$cMvIm%HSE^rDM#OVZ=VcENA+Jz6Y`|!Xk9D_ zYiY#))$1QNix8m;2VD+~!N#ql$^~Zk>ImeiI=Gp@tZ!p81|GJCo|`?j!Q6&p&k1GB zxKaMvD_(c3LJgpyh}+Z~vhC(Q7mjL2y-#`?_>!5JnTn0GFX(uf5;r$g3f|wWMEn?6 zZ~JXQ>KD+Xb4+b5KjqsXe9bo26sbw-6_+&xOBM8bCFSi>nZbeY+ZiA21HGhw{&Lsx z>+{bM>I4+k9TpMUfSV{!oxJJ1Su%xv&UF(3)O%e%gALW9_aM)RWbXDSul`x>05TJ* zVZ)*PlZrjUV|tVBJ0~0cG2%;0NP}jp_cc$vrizQyQKYjJRmkWZg4}R>`c{PHK z!s2;_9Bb6jemY^Pwro%)wS66Zqvh;d^D)e#DSr5b`Ct9gxpNoGE{(VPWO3YbXsu&& zlxHAF<%W4tR9Kbgi*N2ly~tgjSGSI1JSplsSL$R|a5|W{#d*0l9l~OEEb2tn_g6QD zM_e8BGG1Ajt-qqfn!$>k%>xxJ?)hHk97FE2o85*os9xLT<3MVyw9xwrr@i!{fcqp? zmbBYlgQ3yF=x!yt>Sw#v5yKBCF)u7#2rgSFY=VN!PXH@dhiqp{?Fu{qo>v4*fu{#h zhbh4JkJyfr4KUTG+p)>=isRlq9I5VV65+ZwYs|IUkofxAgvMm1^?K$(7XIjLLEwp_@_a7%_GoSbrIOqJB&Cu1mYVlj!%Qbz zl^jb(WqC` zqYkU(G*Z!MJ@+;OBbG#>N12D`eaP^;T0|`qbmIvW{PXbmn&_~A(H=j@tI6-2mRRI} z$e%1cgHwnb&knog+%0qvTvLnr7zAcCG$2<>S?+LpKHDq`>bL8Fn}Zo4mhcrSe;Q)U z#;2b3oD6elhi7I;mUnub2K&l$EbROWK5ku4;k=48YG$`}kxaImYDT<&8zs49ePlQc zi@vylSz9$WMg9KSSljrk5#$|OBuw+?FhP_&kS8g`2Gzn1zg^ft-MHhv7_qkygq2RCM|DfxM%xo{%fL=y4_Y(>f8za=(&c%n6r3>%=% z7&+W>K-WIS)TlP@#ap;15ze-Cn`knjNdIk3s3G)OA#31aQ{^nN`QxXampq3?>h=bd zpz?CV!>%CHp4Uh4wB~b9hvtkqM!~5BbwjLC^Q*1mM~zA-tka4(x4tl;Yl4XHj?q%S z^K*WFe4AyndNX0^x(P(tL+c_G4 zn7Fi@9cyhAk7MKD%6Y5p>4Ps%WEyRY!=YV7m$a$*)DqdjEhThIeNi=B-o}%u5JM`I zPJ_ieDcU6_q@*WTIyqF@GAon6Imi*RpG{;j&34J}AK(i_JMluWOd z_CE*Tl1d=S&{^d*id%GcV7oU#E1yj_`s8?aZo* zf@-=%ONscaJtG7ADdTxLLlSNr902Y zI9&)4mRJ(J>iW8v#9}_SYDozyixUGe(~+z!5v-e&1ByzY$p7MZb03GF^X*m6kI-YS zjyi*IS+-7d8?h$TR3)oN0wwdhg+Hg=1cp8z+ExO;%jKzkP+VD2R9}l##PxG-D=c5> zLjZUrypupY8?v4aO`qND+y22eHT^}x1wU4GCkF4#VBu~M*0;2%Xnp;WDcbG2)CQw) z%`$F-{EXG|1y*-^W}VCRrh|tRj5AV0NqUKAL(Hec1uL;XXn0A%#|6VBvf@%J0~ktH zqY2YE3hy&Z^w)0?jmILV-d#PJcevladpaL=D<*RAUlCC<;VkGye5fa?>&W>`-M3*s z;eQTrMGgxgR%J{@^F-2Dj#{roE@fkjQ+06Mt!RKprI{PQzb6QH1 z8b+Azh7{5L-{t2keR4J?|Mb+?en1s%5|akt5^IlGHMv!9ie|AEzfeyX^jgXjaqv;( zjHC2O-JCpT(4G8UJt!{9FHT9@A^o6jK#r}F!p52I$oeV8cZ(Ml3hVn0;Fk@*)9&y9 z+}c~zSXw51=;oX7p&~+T&da?YL~RW(({vUHYfjqF_4CGaC)1Gu(F7ld0|!{3+lQmk z#i+XVlrBgzua^L4crMUEo{v%1!?9KJSVU8dHpo`nd?Za$z6vFJQr2DvgWv5iH^-Vl z)ty|maP*^zE(g`oEHuO=bO$PmDr>z@>h$(4yZ?Tz1sf~Vjj%n7ul<>eG09$jY2DA7 zQuE*?s@(X=6h6*y)_4A^twm-1+kb$*SJxz*x8Qjq1uF-?$nDFINfk^D4E^XGc}E7p zvvl+2wGpe+NSv$;`$TO`F4A{WDtDCPr9;ciRM=9DO-$95i!Cja+0(N{@Jc>7KyLQ< zZgro)YO<8CJ%49}s&h5SEIa&Q!}~W~I;&TNIAuX>I)709-xrr5g}Hs3L{=0;baJ#{ z+d_*!zcf#z#2?%N8$0RWn0@`(&l=CYwy+}8qx6SI|Nd_}8*_?FRTyGG6ka5zcO0|*+yu0_Ne{LJ`C4vE-^@4L{u)SJd<)8gR;;IU~y&WWZ zkNs;YuQw=uVY;Is7YHysAo?}&f(nE_1D?@W(3Q*PQULDaGA&WQU1;}3hbZEn}LApjpzQc}=dpLs0A zPTDSfMoYM5TJRowqtQ*D4r?Y$J4yKZt)I_<577{5osetGK^YCBtI34r*liFy6D<)# zg^l-#6{-870+gT89pTqM)~2or8OY&RsDGvee(t*BL$=Nz(jFF^O2k#x9sjagg#;NR zpfXpWr04Yi2=ci!NdOEkqb_mN%4nI$KAMf6$s@CE)aY1(X-)4do%j%!13$|f_Z^l* zgrlyMji~5z?~I@deYu-0ir{DFOaXx6jhi^_q zLP9V&&`O>J^6YHWMv9M_9yAh+Dhq{Ko{;?6=Y0Di@lXB`0~M9Ty8QxR4F$yfa#M!u zolY5T#Ss;i*Sl0lPhBoF5)8A$Ic0HJcKF&H%GO9)XHU~w09Y=eYtwPviI3HdTSa#~ z)L;VLyig)rK#}2*MD1`v#K?u1$%JNe!Z|fDUbJddnxrxTKe=1hUXJrkPyvU_f>4p#E(`_u}Nj1K9%|{=Eajo`MSsKH9{L!+{>D*GIMO$^g&Rh<1Pa z0=~U{xxe{0Q|O&RQDVmk;%`^HugeP|=NO+nCe#Yels+zq(cZ^TYF`#H1VBwI$b zcg(#xbNik&UEb3b(Zr

OiuYO&ogIXf=v1%V51UdKkg=gTBr{lj9snI|_XNssAp#`l1}l;B*!RNM7@1-shdM2Ry=c`VoL zDx*aF4UvO3l2s|-18qdPtic$&Ig!dK7}6vCd_p4SrRy5(39Jqb`K2Y;;c{|cO<%>= z)}+R~jfxhDRxFZeEqXa+`E;?0G~K|@a^0Y!@lD*t840h^|8MpFP)6ESz@kdyASYY0 zs(vshNBaE!cq@4FALF(Iti5*B!_yVtmMDy{b`6md%?g@YF}Z^>*HT1Ph>$JA>hlSv=ZCF8xjZYPlm$8eV0;x@BRd+tqq^d% zt}X=4ulMc|OHZZXu+^{;fZqCp22PjZVJZ~+H9C9W?eeP&NNNYaxO>0fjDL@ecKNu% z*q%5m$c9+TKn^bc)j-M)<<3AMkJkpk6VBqm`vM=~?ZJ_WO&kLw01>E`lyqx*DOnCYYJejdponMB% zl)4*t(YFzM{Ncu>+F&9`FQhZjr#H>AV?#P3n_px~IVyBPUUqK1Upse3gCPkqe9yk? zIB##a78mb~d@-FTvhKP-uy>SOO%AFzs5G2js7P>;aGadPde5Od08LIlp9mqgJg8rD z@ESi*cr8)nFcCECu$*WSPe}G~BzV%U=$S5iv(eso>#iWgh@hdX$lty5DVPD%J6Dw# zo?F(u69;1DwHv(_M8~=uUz`^`yW;e2@TN>hT@1~1=EPDW6x*gAJza|jS68&4QA<)~ zdDGi7Hv8un|2!dyNI~+nv@bUtOk>t+{!VhctvA$WMl0iwC#%uc!_Mp@F5}Fs5kg!r zy@`)AJQdHh0XS$F`br9Pw2+Onv>aV>;Z;8z&AR*9(qft*pPj{8mbY`vOpni4Q?OXk z?LIiM?t(l&48*?7yJekA8>NpH_$x;rzE|Dyl_oe1)#L1>@jMKI$MKX8a-em4URUWm zkfQUEv!JA3_08heP{(HB%9N2YavzrPu0GUC^CTfKp>nz%_S=~72t#^{ez{SFK0doB z;WO{5;q$~>#*NFe7e7KG=ltv?^1dXyeKIK!T?XiRh@D_Ts+&E%C=$A z;Ac*Ic(-|Tdc2WHQ0F#r^&Gzsbh^`HK5vv`JA2f=oP}|yCO5|6XL$u|Rx>->T~|A%QaTMEGH4jbXWMBx)|S6G|;9>2^)Oc(Ey@e$A62F?x`l zjRh#xmNt>3>-g79raaU7LF(TMGg=~S@K~K?E;*mojbHV_8YXoN{z%zFdVdd7{=n3B zw?7!wZ380O!YU~_Xv61g%^kZ^cO51ozP$QD zkxApFL;@3j48;Y3KZ5PvLiTVJZ+at_9D7AY6M}Zq*c$A0yAL>t=d`rLZ67?^IGEma z8{%6Fdk5jK=!=+HlZoN+%btj-K^=9SXUUrHrqI2Y1Ey>Y2I!gFM-JNP`0hih7W6C? z{38YhGj)!e!!OLK&VBli+$)w3wQq$+h3<*0(Euu!&w(C;YW3s$iQV%-1HIASB6*7M zATZ3U>c@1g$F^77?F!vNh(b?q|GVPai4`>^F*#;Bn%FlMdR=DtS&~HkXqPsxR_CDihk8Hlw#FazpogZ3V===Q$8QlRX~3+pwILpd`S3o4+HI# z|L@O&Sx$NDURNQ{3-u&>``6FX?~O$ruSJG$k-7M&k290>O5byjNeE+US!x22-Qvvl;VKy|Oz3B~^x1+{+1$vv znf=-R4BrDbb!-|=*nxfiM|`9w8s5bKY zG>mYzba~GH+0yIp{f$|c@4nb?q866LCCkVTaV4H8dV5HSwPLzxsEpT`Pi1F=(~NIR zgxO%D(V3E|llJ?UE53BD+sa?3CBF5A>3VI#GuWwJIB3>LzHi0vCZAsFl$F4UwFD1b z8fqT)Au2a%SVVv9CsH5aR~H1@&*)e}_wTkz@B~m3>O&Y$1V<5?IvFO>XZEb4_zPGe0 zCiffk1P)zvH*|exC?tF^n8E3~OTW3?E@(UL7IO& z`9BLQnxp-rPXR^A6?qf^}0kwa4M3 zx5mSG2PEqZ9oJe;SJtl@u<5c6V^w`t+bkNU`$ufko+cWt=XcDI8$|2h zM1(9@o973o(I5J0lb4}-v!L##kIb3Vtv8pE`0lU^TMkz^t7-MdI01^x-;Lb#r#}^t z6uAV@u?|{0gCN{S_eto)we~IH;%xL zHz9K`33|L%?2uvT*cV|gg^!y>S$`8D%}MOhAo;(?5_+VqsVm1!MD%Vop4+?S+90Vi z#)v^O=;s;Tew|UlZeoyqUUUDi$xR81Q%VM=NLAlk`pVWrbSlYP=Tos)%neUmuD~Y4MvNC zDE8937 zEPHShU2~>MXEvGi6nWDaX1?svDZ|XkT+UcuSBu_U_cwu8mdnQbthD>Q*2^y_u3s7=CnlE z!Zd3=hb7MkiB1RghBgP{9XV}wkNlXo;kX_@(7!o&N{+M+3Eiq3H(ziMDWDMh^py`U z-po>9U0mU}5-dI9?R=FI8eR6{CnMmy^oO+T3OyQO99wqB<1#q@no-`V|wVb+(lueqP!>I*6MZU4n z{j(81YIM+ay8Or-)V(Az-iO=veC$=eq$b}#+fl|&aEA7q-0O)~*LgWHBOzY?kK2dB z9Y?MG;)OnbX2_pb`bOo|@#y9KuA?^22XK++r*l*Kc!9m~?Af4s(Djaa|8Oh{MU3wiCzA1Q88!;P^TBNfcJTO5_E$}N62d$1CxPc zubJxnR!6mK=QKYgg~|I&9r`v_PhoP3M!Iv2{p$b=vZOWKw>{{7)Q$0@**i=ow3Cd! z0>g!)9-f1yWVGI#?6luZ9h>Uza8aS3oSD-p)S~@;K6e*zf}u|}&V5)DJf631{;)zRvX5lZ4M${a0#2t;dPo3QLl;>pi523%gd zs-!}~m^CX+kCH@#`?7i`v5MXkf4_5mF>{m+E(wiJZtt}qM&18Vwc)T(orXk6aP^jr zDs%GRn$)uPnM(ZbC^MvW7uxad>J`{eE6h)YFTOuGZ%D%Xb_8O1=xX>o^G^ZyQ)(~T zdo0WKwnZbOY;f{bFdJ8;b|-Y2xa5G_@s30;H4BfG4L?uv*8zrlfsIyU8hFOeSknXh z(}n$3iQ{Iqn(YqUkI`(t^c5dcu%id(&Ffy4qsy|U-KeruUap%zcOMU8&PsDi;RiDk zj5x!k9!s#VE= z(Iw;0F_6_!259Lx85Op(JcB%ao?KI3GZkukr zX}xxpWlyN-c9B>PhGx;yRaI6K<{MtQxs+bt`vp zg=SuYEF3U#FHy4T>jBbMoi5W|5&N;Ii+ivxXI^4Rf@&MVcfZ1|7Ju&r$>*b6QN=F^ zroo`$@WkadtG=u2b+U{zJyHcPv&h{NbD(||`ZPKHtfLA3kVW=H46K~+F8ji=Y}({v zCAa#r@xyb>;v+JFJiC?~oWv}GN^f+&f@vak-I<^R=IGS|#mv}}nxbJf=iOn|+CsD9 zKnK5q$?P#A^{IU9DFl0aRUg-A6=(@x*=$+=Ns}%kl(0zjK|!`?nkLe(u;mGST?QKg z6|vPiO_lTZmw7mwmktb!LzTelfV~wpk4UEPplWIMBk}FFV{s*LpJSFmM)A(`gDiZF z9#oc_CONI9Q`q6Ti{jE7Mo9NJTfO5`*ukYvmRjoqL+eGEa)tZxbBXn+UUr`p5^cOl z5y!$u&d0;nY1kmC8h#dsrdd&xvIte?E48^mG&Oa--D{!I?o9QA%zit~T}G&K^`|Fj z_9Pw0(?JjMQ{i#zu>dC{yWI9jS{`uPHP4BPy6mfP`>V)_+t+1>llJOlxD6+p9xi1^ zO8Pl*qLHYVbekKHvInkfAyTXK(&d}ryJ=dj??X+xF4K?9Wts40s7*D^g(0YPozFsQpxYea#b<`&veXt*(a|7(K?SUg^{Xgr0dO>$yx?5z-lVogKs~ zl_l(cL9MOTvAtb0I5@T*)nr%MOn?kz*J)?|PI0L5>@c(h8B|}{sWy^4UF6C}7@lD09Hh`D&s$wabJ3RKK=I|!LRp9aPN z7>AZ4v(olj6`w0Gq@-*#tEf-n4Q-x&5l4$5=Y5xO>Fml)I^Ri8Go7>XFI0|w97{Fv z1gk!SMbO3s9HHlGUrS+9#1bX%YZemvL41=HqrMq?%54%DCx63DJC9C z11q+yA;PCx4N4|}aQ&u}tI>9c<+AR$nF!J^^6sK44u85SEj_F*K>+9Km9jtEy4<$O z(Akbq9u#13es`EFwBbr8I;hkH3Et{(I@=V>3SEy>@{3NtieAX{H|8`TcB*#WF1+%m+^p046X{z%lDpAp7?KZsw0fa2nmTt z_P+I1bF1|zfAs{bh=TpUAfx1mT&^ez@7_H+lD~Z)vj!WwuRQV6)hee9$+&Mm(W~@1 z@;R|(O)s_u+~^@ux$)LKWZw2@Xs~(?;qK=W?}&q+8Zw;ul>*@|A*eBRyI z?0k|_iL^-FZq=>^3uIuT06-S&vy_zNjdY z63g&lh;qqcgASIiJ-4vXm6m()aCX!`&;K_SHIc$ZifNdektH{

mF%!`gwhOR8j83mcWD;LRuuCuD z!$FV?`uXKrjIh07c&E3$^vqP(5DCa><|p!NiJoPBLOr1?uFFie9%~J8B#%dDAM3Gk=#J4IEV3kP^9$KQx`{Wiigy!5~2x zx74?{WzmiYq2itkT5y@%-@DF(kafU5F(hCz*r{uZFoShl=<4L1yjQcdNWvtw6_p>K zfy|_!v7*AQRmbz5!yR7>i{l-=K#Xc9USyKCEpeICmE(R7&jOt-0pFfcqbmKrg*xt&Yb6=82o*eF~ffp7n*1lJdn5wiMze83^?T=s-4pb*`(SW*fq~tCb$Y zTNkhIPQz;3W=uQ$ZA+OiLNs@s4HsyH9q1jQxK1ObB-#Inh=dJAjZ={A zrm@c!vujDGc^qvaGI^N`NnJ)}=ZVUSyxm`8|MmRsUeG|BB`N?hdLx05ofqC=B##Ww zS0Voz!_Q)Lu0q=E+I>P)KRm?mm*C~+86YOANBf4R<$6o?#Cy~L^TQfo-{T$l~5H9y!4(rR#V)B?5; z3e#_vTF>o2B;$Ok7nJ~!nlvBY|Inx-s=SY12w!YE+<)^S^}QH=eZsGgW2`i(a>53K zvv`VIF-8=;jtlaN={#{VM*3>9Z$#^i4sY+y~3j*pfW z^_5wrk@qQl`{5vxn#XF$GAf-X088*3c~^n7>nk_0>}!8n@QU|szmGt**Bek4XA;Fp zdS3{j8w@8i_)AN1Kd~w!Ba$#7=!p&ya6Q7it^ca&m1+QC2{X#7j38(WO=lmRn<&iD zSnON)wdsAI(-(F|nh^H!Bim_5vd4xj=OV|Q;?;2R>?w4i;PBnrKz<4M++7A6j!0o$ zdwmf<3)<=N7u;LVuE~NdfNGx`!Z>FYO9oLa+m@u$w5_Ys4zk1ILFD=}WvC)ko> z!%z5dg^jhX=|GUd&2juDA@-hPy=CO|Q5>oH-E^8Ov7xC|pQ7(66vWfuhrYbf{2?`f zY}ea;12M#iwYr`OM2$td=I`Tm(>)l!pOf_N?H;e;C(ZIjD|j*zCd=a(%KrnFKxw~T zF22gMhtqlBzyWUm_zA{%+EIW&+lb+ObNQy5{OeI(Dyh4{8<@4sKlD8x9hb_X)1Ty5 z8`GIxP{N{|bKLsPGEV3fNVQZ-d#@Ni@b@*$Z#0s$P*b1Hwf|bmStEPV$JvH5sYpTM z!EsOji{E_m0>`BGVN{SO)lz{FWDM`Om<>&BR@u%==RY1D$NocS@QtrF@^p3~ON(-N zWa~G4Vtg`PR8lJJy*P0Gn><`#wVf7yGcWJ`nv3U8WlE$EZRG8-+t7#k*%zPi;R(Yy zc*HnPpE;8gN2f8u%brqEpeT3as25f;zd?6h$Ba#7Ja^(CfBf(TCc9aGkCWdZzW(#} z>v8bO>@pgt0ti7u=itG7XV(?# z?#bj?sHx88`*V6w(ayv8_j`nUi(790FANPlykarKY(xq&JEs0=1xs3uznhmbYwLJy z_1`(Br&XUtiJT5@J`4$uWOP&{!-50o;bcpxL_(R$oA7bHPlt^jo z?8%_82u4RmF+4PgUak&QNJUDO4osT;KCd+Ee%&0Fp8c4q&ME=|{f9rsw_jYq-XWfJ zcJrivfIodaU1%e>az&z4#eiPZ`R$RCuDMrJGc&h+#OZwksa8lSQP|ScKa?@?JsB4j zOjkQ4B~lfA6Yk^tTh6oDVCmM`m!96pHx|!gzZieoDP?h{c>pJo2kCU1*iAagUogSuLAeTiINFg{xkk z!2o+1Woj2Dj(?6jvTA4~sc)@h`p^I3;<;lPAK*&0LP}8(=sMt0{*qnaiHES5^ejEI zkxxw?#9@;k;YUC3<;APTEGfw5o*&=i?6ff2$fQ&_23vi0jiv*p(>E1z)hp8(;v2<9 z|J=m<>Za>lS^kOjTs$_8uAu|?&X(NnVL=*Lcw!UZ{mUYb>=Qv38wI7JNQG-CA6tIz zy8Kym4b0r}IiHv}nZ3h2sFqpnTG+T}`Q>NtbHRN>IAqj#&X_fW6GjbSn1>xDqCknl zljC3hnI%p7ZgDezT#^J*dd7_8TdVhS-@bhve)jI(!OiPd@sl^6A!}uAm^K_Z6Lr__q{t<_UI$MbWbs(p| z_%44~vyPi~9^}=kb~4MNpohcJQKt}IA4{+W23K}dXY8uO#opF*oR=mJo-u4s$3Nj~##`ov!XWrybyHD_H zNjb}kb9rdvmwaSQPdZ3JK%mk&f{Q*mze4NW>^`@g- zpwz~L!=Lz!7prs~bf@x{4>>T@o&p4VBuwP$jMmNu(qd%E$!|C&$d2}j^SC{;vV(EA z7`4pZ^)3hY8O(odyu@yCT80MZT{z1VNB8l|muE6iZFS=Ri6qId`;@tS5d)kQT_#C> zeYBN1e1CQ{#oHuFjE%)y_0lLh${aZ1&7G`j?XW}C)?VhPkEC(H+;^E-(Q${g4Vipt zSU`J{B!VxjKf|V*K4R2W^U|r4JhE#g9~l`%l>kCiGbm;-XFmBHKV7|(>AQa9D-Ta# zVstFWJoYgw>)L2;$l+&y8cJ8WK($vaUs_wxF(J@)m0vtLjLs5~w!Xdi#?XUcdp|7tO6cxl_;+1FD^K%m;A zCzr0hKutF@*%oaHe|%{YeH|T{_S8?Ty9G!Y8?W&5#UmIy=?z|~ZtY-?TdtkvYZD_W z7eG;N%U%zB%7T`zeT2U03jZ}Lk)Z?Tal^&-M3BWyqh3p`ritf&e1?&(YOCLM31t5< z^ZD8r>zPqdO|8*FnhmVU+RnwpLn#9&Q@JvE%rs7&@hHF8c7|0g?I6UgWzn&3Ikkru z<)T29{{X(T^)hvLNRmig{TY`uf{(uZAvYg5$*k;bW*ptk<$rse1HxRaK2?y>C2}-B z*a63~^La5LFBt@)$Q>tm@=j?wlks zQBz;YGslke$etB^ASHk*2?*gfNfL{RRTsDM>2Yy%3rOLstB$j=!xx%ZowJh*60Ag# zMCHukFRW#?{_ZS|*`#N6VHQsx+sQW{986cG)yCMm`Z0Or9KQPD4@}Rhvqt2>d1catPB*5~r8XU5Un){Y@*4{<|g6>p7$3~d$M`LBhH zbe2;P0feYvKJMOp?u)&jl6a%)-oGddwB<+8Wl;EAW-2D z#F@+1vaq>xAg($6HHSsIQG`Id=y6AMjvyIdw)ejhZ^1TD^>e!<;EXpwcs*OMblANnW)D8#uG4k2Oi+ zo5l}!WYciV{*~oq@c8kgT=&5x~u6FK9uio&0@ae$w%W%aqWoRi{b-3x^sLwk?m6R&>E zou_iy(4G+2*XME7l5uob%LqZi;NF8cV)P6?`~F&Hmp5B&n8n1_^3#0hp;S6cL5bRx zBmTUeH8;zzrIqJ5{EaC-Dhdh*_Ic<_mUl?1>Ud$(TkPvCqo{P|peKK5|9cCKx_ahc z$l#IfUvPH6pmy?3L63lVj-CHBKV7|>nOBNgQBlIo!&~_F6ZbLDLD8NlabW+4K4wn+ zJrq%xntAHSS2!Tpl^&79`Tnje)R=GOS<1Qft(o+*mESf=VllMw!j^YAEGCSjpZ${A zH8&?NEYvp@bJd@R(@7#wkjNM?U=ercHPS*0TdRwim2r$~KY5^oe)?-Z|=l(W3(G7qf%mk7MIKDFr$wNob+Lprf{>mdC$+oWb_i z0v4HDDEkcdhAFo6ETO%NN+&TS);T?R``DUV1q@@zxeCFRXc$DFF^t zdnfRvRTn$hepAy`e*V}Hx+q0LkkP;IRQ_Z6&s?`*3-_JQ@6s0DQQVsH`Ta{%8R+7~ zm{Cvhm%OG<_NMjPDK4AZlQuH3%Opv4;U%6rag;m1e~Tl-?I-}9lO!F!RL3i;o}{mx zoPfZ9@ymFj?Dny0V?!1-77KN)<;*;GjOklG;nZY5$^{Uj1LOLR;Qi0M!f!Y2bTxUX%iq+WB+#-!`&L zV<9bOn)F&~T5GuLty%PUu*#}~PY9C+P2scet>*dCMw&?K>I=E$!$%n9V)YAlA>;YO zxw4KJ%%W%B;jcN|M?yeCx3DQ(Q`G!BGX|ZLBzp=gx&Pb0FfGKHq99NdMG68a%I(-= z$}2ot(#CtqlUNK*JhS>m#@LGl1ghMk`1B7KXt)gz*;<*&x2H!@3Q+0Zhi~q>(kAj*7o-~I>v)CQKZTxh(qT*%a6WW%ia4A@#v95{AK4B{`<@KxOnauMh8c8*0S|1 zH{6iFtaWd2Kx`NXPkn@c|McJ7b?6i?UCC!*UN(>J{DFU-(T{Fw2}QX*6Q?iZsY-oU z&XOg^R$Bd}07}%J>^J#QzWwDY?%0>kBS#K%|DJ7J{mlnlvSB1q8FK3aiGq+HBYz1fa_N7!5 zsd5YBgvVd!_Z#=}WX5@(I<%ADy!!%2_3)uWrex6IKXK>fTHbxS%EIQ#LS8s~m_NMx zIOCkH=Py$^GA4B#7cTi1H>aOv{VIF<#i(V@?sqxBM{UKK zZgG5J)mgUvij6ErmY({SV}tEz*K-cHo~!80mF@W_2lW}mH`ix%k1L~*m3bF<^2AYY zS^f&sg4#KVTaqN#xr){*ZhmDVz3i;T*-inG>@|2AU-)1hv&x%kqJ^5;t6cl`e1eeNLKd{}F8TCb{vHXqauv&U+-!vn1+BP&)SE9bn2(@3w4q)ynt8 zbRHs(u3v%uc?Ea8f_v@QR;)jqgEor|`~!pF=6Z7vHVHg?j=}i8D{-Q@0d4vw=+vPY zGI~6`ZoYpfh>*xskjsAK9*!V|Oe*Q3*R>ROEJ)((GtzH!@+5}!@auTRPgKG;x)0L&gd_dxS!fzcaA@mhWKQUZ zRBu;ACA$J9Ka?Nef{njqL!&pK)g&Nx@B?^YTpa9WL}Fq;Oem~|(lZDiE;hfpRWh{$ zf}#@O>+g-6*blKOvl0fQ1r9Oe@cc_pVQ`Gw^;Se)UcLwp_J-2ZhPS>ygnDfQGJaW) z-IGQlWJCyL-MX%O4LetSf}hg!APE|W8RG{c(DBwQgA&;LC1Bn|b8sXh5BqYfab(?c ze4o?~i53s5@w zBX!I;oZUxQEZuuQslpA>z1#qk4~h?M!j=O$&>Hk;Hc5~$^a0Er69YRbk&xIAlZtAg z@(PBlqY9>yYUCFbp}|1N%w}jcZLk2>Wl~Vs!qvqYDjCt(T!qV*@}M6R*zMtcW9I8d?bK}rA#B8_XwVv+#8P9 zUs?v2&|Vllu0J*$%)>Qp1B$NHW7g+yV&33TT)zb`D&e0z3ZwdbjeR+lXsjy1r7QVp zGDX7uHljwo3jOCV#?tvC;cNTrhawT%GaezHc3AT2M>t)kMMJ>}tod;>1}82?w1Ws5 z8j-ngHNN>d9kmt@%$YhDX`!84(MW8(F>vPNn0nwOKG<^=`A65_LS$lv%iwwx=0R*&0UXD`9dZ$HEj`^ymh!0VVkJOTC{dX7op=n;W| zLz1vL>j0WSwAR-_f0ueBO5|`0jDcHVIGDW~|K4&KwK_ATP9d1G=+Ah3Z1jz5?w+3T z4Ge^v%7o{Zeu2zdJ!*3HC0ILo_XI4y0su=_`Rvo0OAQWwk^GZllQ-Ek{~ zlBwO1I`v`9IerR%TYCYe8Qby6SA7uw?jl4wbg#vO@!DmaFRX%AuZFfw1EU2U+PA5^ zlQ4B$3QCIz^F3HgLtQ5R{rNZ8o>>R?UL!DZ%piE#-Lghe2ItT;%zN;DoV@sVoG35B z&hI`(kDhUO?!E~8kqt-9+H1I!TMn(!0>Na2R;PmzfbzN>w}Y356QmLW`sy;|6qTaI z90R2k5T!CGq*6Hh_dtxF3R|xLW|-jK>wYYGb`E+6JGNUJ8wB+ki76BJ;+In;xTbGI z{^`>wH4Z^cH`|xdtVRB%T-54}fMA49ql59L1q*@*Cr?kV%&y(YC^(Adh8pB#WueX-3$Jdd zm>??P9gzTU0IfDP*!0s+I9FbA+gPY3V>>?kW&^SsmFPEq97d)_!1k7Qf}+e85rb#q z!C5D8>FrIZEIx&AKl%ziV&BG?P+JHx1=KPH0)oQf=V1>Nw!tE`M^xHOy!7-$g#Oyr zRN;i!Au};GZ9R?`)uFk$1Q)W4(5UGHd%L@LvZ7Q0wG>bW1R=!74GN(Y8bO7?q%nAW z@k5AkR$ljEa>oD+o;ns$E7MVOwm@+s9 zz5VU)z72<2i-OZz@WnSU$5MuGITtC4=O5^Y`JZkbF0nN)(np3(4CNKptqV6jEtdC%af8N=Xj-)`gV zyf9$Qbd24z4d*sohPJr`XU`VEoZ1n)S(>h5`<6Y(E^C20+!aB=LAbd~qr^T4qs9)w zciT>*zO)t1S^+^lQ!y#otNXJ;%`LceU=6-rwFk9=FDA@dfC0gs&rIzRi-{9c@!g)w zC@|_faUyPk(Qpd{^xkru*zgVZUTOKSg#G5`G8|a@EjFL5!+@7(Vo;cKM{FpuMO4ZF z^om%8ixv53sknsQJN9Doz$ipJdm%o>3oy7MW5Z9_cfJ$`vj(+BI}DjH53>dYLn)*p zE-?-FSG2<3FBEF21{nv|;>Rr~p&sxGrVk23$Bvp3xJ9R6P_GF5l2wWNmU0~2y#p5} zrNJxO84|e?D!CFdQK7I`D}W{wj8YGbo4**3O-+GEd#`_qlRr`?&cn!cn~`(468fr( zIFs9eNip7#{Rcarn>AH9w*F%r`0^_pFKUKkP%n%d)&qs<>v1%<0hjmvfX`OO!S9*L z@U!WjGzv0X1Sbx|kfa}wbD#{m#&R6nxeHe(#3I`E)}bFJ)Lh<=9Y>2{5~K)D8I9qw zUcWiR@VDCh)fb+{%1=K>dQLSYjzPHZ(O2;LlamqZZr5R+9BIP%@e?rl{AJKB2F}tO z=j|Rn3`^ckK~QWW!hM`N{JX&5P=xs?QFHBitU6YVoZ~yN>*y5pm>i9pBS?LHCVu?* zQ*1tU4ODtz)WRiLx_AzHhd6h%hcx59iIXw){ArXali<`Pt|zqhwQ%ioKVEw4HB3$p zXrC=^uSy=iv_*OEorsw*p!9 zT8IjJ#E*CwuP+`2FZ*9-OnAcEKL`O{w$Ltq6LRWDfYkuB=Ndv+V?`Z?U+>B#u zzs9$lPebDvi7}HVA}PS}7C$0D;Shk)a~{Tt6X)>7{vzZa{28CEOhVMFGvRyJ{R%}m zc!eU-D-_0hFO(#Ig`ZAW-iA9ObX7T6|JBFXa;6$R7B0mDqvG3JWZzhqQ&evZOp3(u zyyIxH7}3()0<8&!AZR%pRD4jv(Z*2Rc@Yreyg!ji5)v6ZWkD-a(Y0$ZgVXwkwT^~RJ( zp20)odtYzGr*a9x;K_3_bi+2}oveknG83n-He+nKJEW4kJ;^_iNO$mx#OT=%W6`2T z9ex%*^dRQVo{mw2lHlWX`(fWQnG9-`;?6w`sX=x|CdwPyKnRdXMTocjLYc}7{Zf0t zQ7#3{&B(h_j24Zd3zfuO^h*>v{L$r`%ha${{n{7P6@5kyL60Ek_TWhk@6cEzC&t3Z zK@Lkx4z_IAgPeK`oIJcCt1Cu9K|zO~;?f$}cz8l3kpLiCiq7EFl}4D%_^&;pOa=eM zQCRr&i+KL=xk!ulg7kmGK>53mJ|UH>AeUN6FFW@r^iAp9OJ1UON3Tf>Ff}C*N~r)E zYH(uz5nO9*yKe9VsS?U|OCu<}(6et31iC73;nfuIP8g0^Q%51lNeWo>D9z2l>GOFo zfM}`8#m;qGP^fW&zn20HRVAI|ptMm7TPHWzNv(a!%L*W`kik`d|W(ybv{f+&URS3MLI2md}jh`TaolgR0%$ka% zAV-L>prs@i$4_UX#?ZZ^X~^1vpSGVxok0S>*hED7d7{hQrJ#Uc$_R|=6An8Wpsmiv zwyk@RTdwI=B7?NRVlu%DKwEGWKX2NDOQnr4UMDF9sGNMzZ}?z@I>=$UCxMeduCTq{ z^Y}k2>v!`J0fKP*;ie>EGMNC278iE>gzYD@QLnivnIx(Z7}FoA$-%G|4&IAVkwdLk zTAi1mf?s5B3`&cFoy;n~c5Z>_*((_lzRr+{0;JBqNa)=cu^~Z->Ddo`;%*5B1rees zLJ$C&3}|Vnzl8|7E9pR6$szo__AqLsj_`I=qPe{27TqjFRlOEYULKGO)_$KAmyaRy zS~msXVu9IWhS{h^{*iU~`M_l~+`Pw-!8fTth9!n}>MtwE>=BnT3d8z_LID#RN;0uy z%YKyUcuPOlHk|)uKMLuFenSVsy9=-?DwR;SJ5kZz3lSmSu)QV95acSj`UJpLEdmgs zviCz&ydOHX36?m)$J-flnFt!2P+n1kHtp>XLFyBSk^OtZ%U0Tva8P@o|AbkXHX;%B zQUMl26S7Vn#F^|WSOBzD7vh(V8<172hukp={rUw%)uoLSVB-;m36loGODTaxUyqC} zKVje1y7qZ4-Bc|>1sj_?4CAF4Rj1Zt<)+hUvJ1kXw7v*%?u=<<4!#&Z_Zd7fe=;Uc zUx3FS7zuYpcZp&VN`(Rvi3mUfXWwun#&_-mEVJ=I%Gf!WIwTwl0WfKsaB=@H$ZF~y zpfJ{4#*VE!kzL*fwUak`BqzW@+V#5BUXd6)b|gX_Brxk5abf>j>^N1~eOqca!$J$p zEtNR7Yb|zXlt5#&-0;Ps8sURRA|=rE?hIeYH$uUo?bvoK8?6=_1V;2gM36(5_*9Up z5tA|)gOWm_gc;2x7qD~l4qR!n{Gp~?R0W`KS}#O~1|zmtZ}g1tza{A_Kq9sm@t8Ge zY;1+z*fF6kh$^Vs^Em)#2i?GpI1% zz{3)$0!oF{D*tL%#HFOd@76*li4y(^_u;{r1L3BY0W_lQbUKa~)mmpy-Q}ZiuE3EE zt8wyL8${a>r1T4e-E9V&DV#zvcFJ%B*eL*03$pjD$BxWW=*-s*H(Y{JFnwYlILQT= z+RAZY-9}ujX}cw6Frh5#EY9TCBYOBK^l(#l-fE>1YNg^g6J4TGd3_d$%rOYP`^LlD zz9YU-*gL`3&l_q1Fq!nIt*k=FQY}JTdKDM07Nf;P2!aTSq!ZYJM^az(^mT?zAX>^x zQCw9E{XJk_U3EV8Zrp&3YZ}=2CS&NpkS^z$2%;2@K3-!bPEG&-AOJ~3K~!-3jpVsh zDuY_B#D9k(Frg|x16#K2M=d?z?@3ga7IuoS%XK20ot&T)0hn5FCHo?ZE1GVcfkdT~!5Dz=(tMagS%Vu1vCAc zL2dm)ROep8rIHHV4B4%C&bn(Vk5KeVh=F^DVaOtsHl7Fxbb$cCVroQHl?E362Mh6N zL`C)?eDUt5IF{cC`@r6q|H2!1>+gTXGxNqE$VmocQ!%!GzZ`4Rv(aYm?%G5+_(vga z@Bnx!MbOxSyo~)ge6F%nA&l8jhf_Ot;6jZKqS7BjN2eml)#i7}9Su#$KC%g$4;4TQ z1wseT#nX>YMudl5C(cU{q_Fo4Me2yr7|C$D?nL5gG{{9VEcPJaJ78(3fCgSmhv(P)(xib+hNMPgahr~hmVN@?a z$Zlh#q)HDAp7{Vq_4LP$9I*(6gFgljPD7Zl0|c7T(pZU__PL_g$AF>}d$Ik{c{EvU z5I1BNo_cgL!rg2-#6l9K6XFI=#{3y+a5a|U=%KTyY1VeH(6rs($gN}@6t?b&@0$W| z`&+P(M2(d;g$6Z7aMA&+SVdTt( z809ayg_I;n3k(hUSo8i;yuF3DV;}(GH_RI~XlQDM!9;)tv^2FqrzdWKVtkhBCoy@#=1&eDJVpPF#_7cV>p~q z2{RQq^xZqi-)z$b@n(UxsSdV2{!mx|iH#JR<_4JH)n2_SSl1y~g8`L|6YTA!|1%pU z+;E^$skCcCU68>ic>wxH`QvzA16qx>xO$}!wT)(YyWed-S%>VSN043B3=357@bZAY z-R($DMH8C9Sb_xh`XY?646A5PVo-2igyG+CWZ&VbOts}x?qF}eb5;fU7@Wu-o6hFKW z)5fGC+TRTdi2%FczPR7d2tj)9B}{_ws|-t^wsV5L?d|1Pv=79zXWl{lxH5>Yfe7)v zQ5|N|Hlj5D0xsp3LvIEk5@w?jrfwr-qU=;UP8K)7tkGiI^1mZvT^EUanxSoKfv2B8 zED&I8YeHj<4v4b7`%u8i6A>Z7aFXSqI%hxL`SUXnac7 z77|nFI_W2qt00%Pn+HJxiP&|3qf#iWL=p{XYt_MMFvj ztlV)1O;vg{mK7qWpaj}JZm26ikE5rGVIW}Z>We^U`E8+tC{x3~*I@MUT!o_iHni1T zz`^4sn4RDWX}1S`-zm8W(9(<(d-fr#vIVxGJ_z;q>s0;wtL^lPABCkK4ua7lK(16m zCft((UEf(zBJF}mX$RBp!AKd@AHLhNkgqYIu_6mOg*v3U-R;dMGph41GkKJvpn;@}6SadR?gB)?`Tpe1f3UKCRCYpx&!KoWtF7*jVj{rBM=houNfq&zr z=bMnX@Bxe;+#8|Z4v-2WTq8!~u>cE{-7Oz9RN?H&vnZ))1LU4?@$`oCZLggPP^20SNK?^)@kK)HdN--X&z@ z*FbL~05gmRGrEGhLWiV^*tLGkl(2H-gvF$T!FUfz5{VLV6JNkLF%!@zx*#~j55GbZ zi&2Z3Yk4?%E+5T$>kUn#$pDMxkHhAsx=b8AaTU#a4GNBLz_KTfc7YQJv#||2XCL@G z5Q4%1`ub|Oyk`s`+}0Mii*LmhoXE&Vt;P(w zlPx^GyrI0kLc%997CplKaUka+w9OUBJaZbw&1r~p6mQ6gaKo-hM6uJ#1qe{7l zm+H09H{>Aw=v6$D76Mg!;iA;m9eu`4$H4VRvFUOXDlYEA`h9bdJU8KbVFh(HIDhI0 zicFDserR;pNtv5`?cMrh2qGk1EnXslOewc&jfF7k^|%!#X#6&!^b5@^mg*b-=}~AK=LoUB&Z+zNHQd7hm{WzYxuB8W=3NUTh@Xy3bwML{XGL zt+c)mX@O3s2YK&)ru&Zty0&^$X+$^%_r*g?UdJ=D(%@ku#qhb0;r?^k_+ZOngRZFxr}yncL8}>3yI>5RG66k2ZST0dqA2ZTgGH(A&Uzwsfv=Yf zlyVVl(V(ub9&I;snZ|M)+IJj9wQUg8fk+!U8c7kZcicxoltF%nFO@sTAhmxO>^cP- zA{?Au;Ao?Uhz6KU2IzG+Zi-P~gL9`(BCo6!GP?jo_Us9p=2GN!by19Z*!oAq-*F4B zW@VzFxeej&>TV$f;ij-u5JbpilG`MG9MNO=2*iH52m8xRFf?63W^N5}u;S4hK zu0c;J>>NDc?ykOlydrZ&LP8<}JvX5wzXi2LmvQbwE+!=PgY7+t)16{OQ%gNA9r*

F@D& ztXnQ2M#jc+6zDcUvXPXM3hpvv!FG9hUz?l59Jpw`I^KGU?7QPbxnZWsMCZs;BBG## zvWaK)sX(6&L8XQ*Zu??Yqb44?1nB;1Ya{6B=*SAKE-13bgbfG?5I&(>SX}foO_r5t zucSdlLJIy!Ab{G5!#=iDBP1lGhocy8*cA%NMTdohV`O0=aocgesG}YDyShpU1cACzjza6YZgjm^epN{P11n&C#4}UQJvE63f&p zwSYyvA^6Z&$0NLtkB?bW(Zd0bbO}P<-fg>>LEWkH@4EElF2vh*f0vj@pBx;ZS&lmM zIc4-2jLyuEH=Z8x#QO^hT&Pu8az8DAG3DorIZR~P+taxf%DNo~mv(UQpC8dzxP)Po z4QgbV3}52l$QJTidB%xOV53?_yNDux56P(h*<8w9b~J2DO&TZB#cn{tXocV}=urZm z7*~d{y^;e>wmms(h5=I&M~0Y;(--`&Gmdpzi@3_2IRdipF`Tz=;P zuOx{?U*z7=BRVI=Ngi}5`tMI8Y6^+)|M^VD&60c~`tMIxX(t&x=t%KjA4gI)@&Dd6 zD5!wCLR|3w>@2JqEcM~P_a!3Qm#~ON{jXi$zn4?ii}Cxv_HO*}|H-NQ)<69UeE;Tk z_4KP(J+|eIjqh?Ng`U%PtshP%4;){8htj!qG&nM>7%_UoR{RV8)!Po#I0+bPYHGbR ziQEq!E0G%|Wo6l8CPCc@KVH3Bb;X~ap8hG3E35|$5fC+RuGLBcrlu4z10=O;m5SOO zmu{}Z!;vaRU|#{<798x=>OCLWMQ5RQw;{A!vTCk#f1ipeDeMc)$oVJ$94pzjgpS7= zGvSe%#$K5#KFIH9S#K^K8ybG8`XDdd4t)OJOxN)iy23@Z1)eahEJ^-1iIbGmFz+j{ zvn*9y>!`-{LPW#xgIw7RXd?LuQU)53+)^V7#SR#bJt&qaZDio}2sCIw21Z&>tOv9& z-8BC*&4$(0%t2R@v4;FPi|(D_bDr3Nt@?ZdpdvH^p)?|5-Co!gxHVNxjWa7KS(EzIel1zgn~efu)phoW^KU6VEKrk-AH ziIXcYZc%b@@5KX05>*H8<$3(ceSrd1mU*ID31r4=Ww&a|p^qqzH)|-~K?D!0a0_e)?rv^2FHO1uQZOD#?`nG(X#1sBB1lQf z<|^`ou%17d5-l_8L%6XY-=xy(Xo*~%7)1(ZA;14h$ zZgl$_s_KP?3`3D{#qT+&Rp0E=yp5Irl0UJ&L#O7KoCiNpDZFvpNVvXdVw5QnN{jgVvp~nVl?UDzD`DsD z9g{)vVHmmALtOju+kx|D4sUUS0SbQT@4A@lw>jr3WVXjy`dCv)(G3mfDpa|hJ3oeP zPcJ59Hp1DLKQFV4-qdOTx*QI+e6}((BXDcG=GTdc;KLTLtAHI$f0H^CK{N&>ngP|F zA%YrklhSsJLqkLJN>(-xhN9gTR8Y>~F8i-o04IGla}$a;RHVuFNN*VG_SZhlW|AKydCVn}-ld*ex_SCqKG+UNvblPaprKgLSGbaq6s= zer#|sAOb{<`?*3Twuw%iR-?@ePXP5#CCy7NN0y8tP^T6f9T9Xfi4q20qEqx{BrpT& z!7|)KV>Pe)m5z;#>>*t^;ai)>J4a9~7tEQnCV@uZAF7z23%v#DwC5dN;#_zC1XhlJ zL6H>9J=Z(?VFaVq@Td?~^lz3SANAd`NE!+?=Gz5(nM~O~o0~bi)H@Iij4`ts9ZcBn zo}RIV1!|`R)uO4ti;K8_833S~t+US9oggVuLa3^$8dW1slhz+e=glf}(*SLxw9i<1 zc*54!bjKc8nVF$S&VR-Jot#9sYe-$>*bOpHCUZJjriI9uhXc^LwPon1o7k&a^gPu$ z6UX#(e+8bIpV0Xj$IvC^>u1bAUoc)a#YMXl%@)xRnN({}DAWbWRyYBBB4GXc_7~#+ z25)R^WRRjc3q4a)_=pX{I0@aJ2s|KtzAtG@h>u^tHMl-r%F_R>K&9*2@aVP<&1 zny(j*Q3C@51yg1Wh{VGU^4Q_9$IC4UC9Lwl(E)0ucAK#?Zk8nT1$Q{8D^$z70hBIm zD-ocCv5`5hEAxu`Az$bT=SdZKaT>cgLE?i_*Zh6-_iKqyQ{2c^3>3BWS)Lfj^On1# zVsD|%lX_Mu8#w=r(}^yrGdQ@HG#QWJSJaeFV*~d}0R#XweI1=w#Kgo|`j$>kY^}Nu zc6Ra($iQhjA1?-9QvxlJv{6QmE}^2a5p?PLIkLmZ$4A#F!^qDcsa68}3KI*fy|*`L zm!r-7nr{D+D^?zCd7ox)vCzG~PK4VNys|A2$(L^ERv(`2!6#ig=aV+;_%I@Z@EA8D z{fCZTAc^uqhBxrc`1y;~>-}n*lSO*jLd!pOAu9OyLKatevE1?DoRW>C25GDp7Z%+W z@{2C4B%YA_2x9K192LZ} z#;~E06+dlt|D+>#M@?94{%!Vgc;oT9DX@53T3F~AW`@<@b$k)@Q7bijb zvh1y*r6mf2FkYhkUr10_lj}Jduuix=?W3cpFOFiYW+L}%LV0B+)C3D7FK_q@VcX6@ z;eSA#Db*Ari)jbUkCBy?RLxtN>Z3xD;^z5(x{79%p=fDnglXS%adHM*du-rjNj-tY zI2}jFkDr?|yUF@GzDWoOed2y134zn{`QV19x?3aLbcTP}^;NcC*lVAnGW)(us|mBZ zkYZ*Al|{m4kjM(6Ez*Gc!Kuv-$FkO2~l6X*xii!Z6_4V}y5jyp|-5@#L`E=R+By-F} z2Mp@vVTxF|2m zLM6=@0E^2?u^=1`JyfJZeBN=%(m6%=jD#I4+a>MR-z$F*o;WS=(~Uu3)NswpHEs`e zZ}(Tjjiz%!!*)g2F|y#Y+DIP{K6*CTtfmWYd&k%^QkImVn<}6{XwZ?*lk5#D?m1vL zXh>fSaBx36k;Q-uLrZZ->@xc8r_TDn8%B}AkU5JEAO1B6| zNQsn$fRrGh0vnJ91C;LWZV^zC1`*lhQc}7_S|nrvi%=FwiFBX;<#V2K-g}%+?>Xno z8RNOX+$lOypm036(sbdc zuJviegysjeuMg^!H{N!>!FgdA!K$Sq>QGzClwj5K;bLK(^e*q)dF8u|s(1I)##)fE zNU`2@(9pq1(J?F6=zq{HbxHX3`a-Ml z5%~=jkpKdPz)QqRn=aHsUxd_t_ADBxcN}W&-}&>=ZDE!({;`$J(!;{aX?OnAtkX@y zf24>kF}1`wie}=b>G)zK4Jj@Jyhr}d+7~xinN?qGc85F^`$DDG%dJ)-#CjxK=t38s zV=c=Z&hE1I%KiB2VtEF4$FmN{!%yOw_{qIq!xpH96mA+(lC!y9-{qYvF(HS9u)r&E z4quWIZm}FSy>a23=kns0p64ed6v`UC{AuIz)OX{yFIEmwokdf1L#8ALg5o%# zHenrSs_6Bc+MDDk;SgFviJK~=Zg=@jC#K}T6rDlAhyq5)FpVo5v<~2u5j?-Juz+l8 zEuxy}?tO4BZTxVC-Apo`LvKYiO|!VPv~%O;{tqfBXHm3G)GHbRsEk6{*w`TC+@Jk= z1NM7@ujbXfpFAO4Z`fu1x>Rvn2D^JrLV}L&(knreTZOt8r2&Uo*xL)5`nuD)ZxNI+ zU=^09Y0maV8;~^pqdfQwS2k#ud+qRCuZ9ADW_9fumYREmi|0m#e`W^J`qEcP7)QB$ z@9ib27`5DQcVaFsDY5H$wY%2cCmW~2(GMV;g-+0zqKErFNr_m+sDZI@d#wx3{DH_% zD~hwPv$7I4$5t`b$gOvq`jv)Zql~4?bHI;pOmhDdQd@cfH9kJx3@mT^0xJr}sNXU| zC&j%3r|8ikLymHZp?V*@(av;Zvf`4{@Z56PsqEaV>!W^_G_ zBUes%oX~U;*9K;{9KE8bg=U1#z`@mcxj?`1PXkS1JOExJ$%<0GZqG zabgx%Z=7-ZHs~n+B0hrz z__d}x9$MJoNj?jNz2zab1-&KMP)(Qn`}zPUqtUcy@x;Z&=VZKACn{^mUPX`DisPY1 zp3%TaqGV>4Wl+Dp?IsydH6VxU&!C!mXb1=Q?rS(UJGwk;jpKI%Zk`^wB&&1P8h&<~ z^JXUwuDS9~l_`yLYp~;Kvg_)v^DhOD(p>YYPCZ=wSWEuMe)lwDLa8iTW_wfOj|?ca zDc?;JWhAZNo%Z>oJU_a@|EfpQX)e^W{{8P-ze920c$Mds!%}cwdwcT<9ns%zf8)2U zH1veyw_{#8%fZX*LB;bJOJ&^pmz1kDU`4!~RWkA{>Zaz1*oF4ykdNL}FiLmxYz(vH zV3ao39`NuCc@FtLAO1&X!SDHJG)Wr;@|j$D{wSTN%(%f@4DCB7N!Fhs{pzz*>_R3? zW}N8SHTd_}X=!(U7=&9y_Pi3}c3*?RLD01RwSf$#3L}^1tFs>B7PwVwdfcSH1Et@|2S-`v&N0l&O0hLu^vV6bD(=oJ!KnDW{(=wx zj__$^-ORXsTvFNj<;(feu`#N&Ci}^nAYkgRyi9tFOX{Dmt*wQ-v->&S*Wh(d<7{d> zT6Yl4yKhDnU+wm)mdv8eIK;4lok_@c;KsWh;SrguQbef8EZy&ulkNMPI4Wt63l}cr zN?dST5GWBVHER+yy`#JVqa3IU7t>sTspCtD-m{A7pPH)kK$n*zPbU~v?SSos0q9~B z&WqI+^CWlmjO~j-I>8xKWaG&%2|-G#*fG;`E|0NCU(?#uI%C$p-B$mapmK#lXzAhp z%>1d^0^LD|o;JpmjVB=I?Cn3jZjFSOqHILEuVrpc9VB=+`X&#sWUVZYrWZXntK2r1 zB@s9{c(5?3%FUXu;;luj6_h(;5B$eb@ zB_kkBC%_9so-m)ajTGry&=c;gd=Ih8^itt}iSw%Q-4}s%WbyU`{rDmOpSHS=xf_*k z$@cyv*wVH;+m>!Wn20rN^|>Dz?F!d#uqTrNqaFbQ9zkJp?{ZJ8a65ok__T@JgnzeF z-JNL;!p|+kNC8xVPYGs6WF7|#NhC{8U!P|E>B%8rkngs%WsL`X=Lu?6Gwa*xB*0%Qc&_#M!-+VA%#%fURPYtx*ru+HF#qC^4#{i0@wRf9C&W#%GQP8XZa@jB@ zOQEefyq3vk!QVQku3dB0g0m*9#hmrsQ%i4e#`*dA+_DSxo*S2q$CUX}02S2-+KOLw z=Pw%J`!A$)4Pbbg->OaS{xYon4U2rGA0VUzh&|LXm5q(@uw}M5F&oHo ze2!-eO?<&rz4Jw^hRG$6ERl6GZe>p=x9dZt=Q-2IctPqGTp!t(mH-BeRG@nrLvyjE z(!lCn6A`&g3-3R}SeaSG`XN0%ee=hGn^(Q*Hd({Rr0itJm&Yg>mkB=T@g_A1S4g(6Br1#eQ#pTQJNg&pzZ3x$zB$XM`(BJ{yMLgzZNmR}51KDokMhSvR=3X9@7 zBp3D@y3C*aG^L{dL@N5^W-yttXs-hQ53&}qM?1XKq6hU{8sqxr)@-1cwL|Evqtmcr zv_zM0&~Sb*M={q$ev9w7N?wq$U<8t~s?akwGHR=|8H2~GqN)mjOW@#B2|P&)`2Mi1 zLqzp2zan`GKRg;9vMyiP@!{@6*tnf5pWDsD0r+082vBrxS7PI>ZCP?W*;p_V+%*7C z5(0Lt>z{m#S{^HF0|IGh{9*0}&;W3TA@9yRkwvTveM232bcaoGf3Ekgh;LndbFan2 zMaeUFO2>Ie9e~3*$Iui#J#C8McGtWwqMF9_o|NyN&cO?Ve04zhm4+R?y&998V3jFf zB;0_n-JgB~f6NV8Z+WT-(X`T_|7|QYAlPj#Mkw0unRjLcG^Oli5eNJs9)Cz_@SI}L;@HC_dVXm3be4$HQ8>J|`dF;IJwJ)`jKaitHuIEG# zFO5Q~GnYnB#h~F42|G!0@6_wNvR6kr(G?-FrCpjY8hm!6?Vo{IF%RSbyv1OTL+*o6 zl9y78eg!j3A5hFVKb@;^tVUtk3VtGCxiJ0NBj|!e*P6A)FSxWk5~j zirwG(5(72>nf)J0xh@09!ZfL)#ii_ov#gKDD2@FCtl4~mT>Qp{6WRdSi`S{CLBQ#4 zhJ3!u8qU5adoQTrT$v0x%bCKHb3b-vxV`xse#v^K5bW{ArSyuf{rvH3U1PjCTOuL1 zDc)5o4*9GRHqE9WHI?D4LLl;4NV(pwP`en;4rd7-V07S47|TYU6|>b)^Qw&_%l!i0p?`_b*j_;98Q9kbtaE zOU;X2PB03hD=N-DrvDtzEdOLBaGb7#sD#fj<`t?MV5X}5+@<2V=abBXJUKUf2&GWs zIvlSD*uo!vo+Qc7zV^hdtHzIpFfcKZQfB*`j^)GVUBBM`{#@-sgpC8U0obd{CXW`p zYFw8t!nx<|?Tzfhk>ZbmBXtDoxu!G*lw-sN4>Vz1O};j-zkrh{?)ICEj5vqV$)*6g zopr2Gwb_`pxw|_p%r)rcxI6l+IVl4@|bx_nVV}uIL=b-$`$NxwcT5u>s$xQ z8{yx*R;ViyZ&?%jUu#G_QzgC+vx019xP>idrHcOo=q+}EMlUN@Cjm&7_Vou#zY)%l z&mWcA@XWyRMsrto(}!Ce?Z3qmVKlv@81aGnzaGHbcG2&HBE!OHtBwN0mE=_U`jP?K z)?4rS(L&MWl$Szl1p?G@Ws44~=(lAl{aKpB8vLfBbZY&wLug&~sDP^eCIP=S*7OUn zvznxC=of}M3sPrOCtE~zPMjZaT+62T*S(*nD8fR4qt@wgMp8smr}$m8v!a#l(mrSO zE8o>1rFrEpZ9Y@_@ZMUpKeb6RW|&=4YV+4dmH+mR>xH|W^3ptx9Kc)L!!W+HHq|4p zK3?Vgm4;2`?pR0{)~H{DiyA>?9>OIh|484QIqI&TWZ7Gd`Cs(W|4~;#9sb|dl>e%$ zIhFANWWlKTd3j#Y2Z@9CtgySYlJN%oT}~9Rj(irSiDiuVO&5K36AH*I&XQ@QT#T^_ zojxA7;PK`UxfHm5xUpWk-0H*(O8vzFpnGurPRv6#!|+cJFGq8WEl7bnMvkdcmNUZD zzz$kv7n32~3+^n~QyCV8FrT)-~-ol2>_H+yfVK??~Sz`%e?+KcE29Ug- z2+23G&N)=qs1YvrD|Io^w~h74tmA0{T5HHA-^KpXO$Sw2 zZ<(Xs7@7W~qpWW(7CioI)fq9u@{4u%-c8nw+}F$(bDgPnDP3$Ii7&{WPrusnO_18v za`Q2+_Cgy)CN8QpLu%P#DIN15fZNQcVxB2H@pElVe7>$ANt+|)s3ZNbZuK7}T#<{6 z@AolF9CkDa)8C`M;Qg)fLfoNSH5v0zeR0TP0A>)ccu+a}(Ht*J>$g+1UNR$we1USQ zeq_Hosgy4G62+-+#{4D2HqL9EdHbS;E~--@nisu!z$wAqvp{Oaq9quHA zt%a@H5*>W08G5;-TdJQK9R1)nI6RWA^|ZfWZ-)|~Doobpti zBSKZ#buq;O7;FFf^@TybFbNvu1Ui}Z`b`9)-NY@M~?FLJ)uGYAsh^;GM$Q+5nFK!jJwQ2$R&C8AgTU{zhD^o*+JAX zC|1d%Ofaf;rAxVRT4vo(`X;|*p(4D&m1cV3-$nG+^6G&!i?~y^|9PsCJI>qX?)SGa z+a3+3U5D#U37fcqm`rI*)%?LBx7>ad$l_zmPDz_E;=o+>K3(Lnt?Dvd{C2-#>*|oojazIv*y&9McdHGJZEoVwDl!_;4+;YI z@mSE?n3$L>ZA?nUfbXA|@dD=-3_|x^ON>hN@KEq)2)?y08MhXlGrac2=tZSvp>Fip zx_+r~OKWTJBE6ZB5vFSTc;(}O((Y>nnQv?8QVr+DAuAHrf?D}^+c^k_MbXHByf^Ln zDt7o}okGivrhnnh77_lS|ChXi%y#rQr=ipLnRT#SYHD78#zn{HKiXVt@$7q|BV&ro zwY9So5L5x>H^XCSh)ymO_x`|fzhMeVIRit&Mos)~eC zo7?mVL{#qsw*%L)u5-T-lC~jUf*s`QQmqvvyb?E91V&+Ari!BrXE*<&*^p@6cs>{* zjdIf}G0vsyoNrw4eW-$T?r`Gxtf_-9{!zG*n9dZ=j{J^&T1zfCNiv`nsEP=rBzk^~ zX)(f4ZvTe!T7a5Er>tZE7mbj&dWPnU2s!(V<$SQxYBc1C1$Pn#KT$itw;!3ptN)U_ z!aPVh@Rf|v_`8kSlu$jDtcT(9-j(m1a|8;{txK#iRh}h#hsAYC=`Ohi&Wl4`SI)^! zo*hM_bi>cJY2dv`h$Uhz-+n_gEUH{LGemVYQE8r+piM)8!TBzSQkj)?nKJ2FJ^_!l zig3#5fQ%q*^{x1*MZsz7i@*_YO)AH{x-FGK8?b7Od&|#K=3?=#669bt&P;7}-=spL z*MCh4W!?P+{u;x#1v5n+t!LVBYb-X$%rm^$sF>a{#3v=VThKwFFcw(xf$jvKr^DFHeydc+M?DT zlR5L!G~f!$ddo!2A+?G2@FvAz8>kZist_Zh|08mD)_pmpB;}=1hGMf*3f`Xf*|vzk z!+pmGJ0>>&J$BdzqkV5CjG6k79`3AC#Z-fVuNv#m1lCm$(%o4Oo6H~oWWDcT&u_jz zS2k1q%<)5>!T7dGXt>7wttFsP@6a3hFagdOM?dt0O2ZD?tx7{uWRHQ>l&GHnd(|O; z37q^Cb>aZ<{C7TPAyN)ZwE;(60kI(IqH{fA7H-<>5CU%!80sVG%zW>!xLG!?&wne_ zK|nWf01{36nh2K6kRI6l-7md91FzucpU!h(|46|9c`l?{Ar{QqNn^g*mG2*cQ$##} z;I|BgAsp=q%nQO_%@W~#J*MJe^`xPgRB{<}$n#%@`R%f_6LD<@5nkc01`(Ozg{&6S z17>#q-gohqQxjLKBzq%hG+=YhV?w^7MB@hO(AZf-p95bp=)5{QI>`Pd?Ofyd9(r0Ws*KS{6K)YgTf3^*2s7a zaANb&XG(l4ZY7@kIo3~U%Rv0e%}^lURQMu}GI5El{u=yxi{eAC_bE&tQ&!z?&uEQ# z|JFUKgUQ(nUA;l6J<+nb$Ybia{SOF=B}UbbK!VbT2X(OpFheGfI>rhke)DkUI&h&x zsTzdP#?%3P$hg&2;0pZ?M1cVU$*pm_#38ED6sj(15K0;huj%RSP$}O2MjoIsjQ&8u z#%+Qah|*beXgbk7A8sLU6BpG$_1fF_{5!7er-!8psYM>;t{&&YEoP#u*t&klTkt48 zDZZge^s3)jC2uq^Q7{{46M(utCiIFsLsM7i_~L}ebJlbvi}h>obxrvgC%>AA+6NG` zcpCI;JiK|z@5^<)igfRFX+s3if3WhwfQ-oR?)-k(t|%(*dxg5fkZ__i^~5_qcE;I* zq6hbjfX>NAKPE#P1VVcL>E>4M+9zW34Fnr4XsL)`J6?Im{={B9 z4W5LnG49T&$<|2A6puI0lB?7-f2D}>){3uj#b8<4WcaPa{8ZpSFT)XFYSZ;vFES=I86YxErLNcDj$>++yt0n5d@~UQd3&DTE`E6 z*AaX!Xib$aCr6;^+_ltZv@-3@-&_Go10=mvjVt|T_ZyQL@rT?j=?N)9g}-{ejVMrB zw?Fcj}3A4mnJ~MCUx#P_jn@v z-4fyb*3i<5mVGe3e;}Ms9Uc*J?ZypW@!jz%7M#_vCLUiYnLT zE^gTG<>ShL1&o>0FFqPGAk5}-)A+#wEI~x(6zwB1KWM_}g zjY@ZrQt1u6wytk}nqBGWbQNa^i`-I{gAN>j*(v7;!feH3y1Uhe47t7CVlC(w*Vns1 zdg=Hz3s%6?)YO_CdNww?64?*mZDq*#ds^YolqF83T+G4Fjr=pnon2YNcD z&!}*#v@ZmVIiY2m>i^;6BOxI-ceT{DEf!k_rIzP<*j6(q;O=p!Eha9W5&zNF&s&#d zYnW727T=Mu*!!LHI~=d<^|yVy+kE|z_*>1@mZ<}bI&W-jY~+Ffc<<^{x*rMOycrL; z9POEj>ljl9lRsi7F%$wzQITJXbsP{ILl4hgIw8B(^PzpL4mJ{yWFm38-Q1o_MIm+f z{|6cVf9p1;Mff@}LuR=xayS~@Qy0u61EBcV1KV!-y^1B^W{DVc7iApGQ5B;`8@Kkf zE_@O0m?LPskj-*;iQ*kLR0KcKK%Gdqg6(welOXsyHHN9BUuKLwDH;JDG#6LuqJg-; zhM%ThD|Zu6@x`iwC|{N#l(zhdSgO-Rlnf}41n%wBPjQHzW?yIDXq$<1`%$(d#>cXL zGD#)xBVNI--?})s{yIV?G?SDk^;)(ckyO3hf`(^#1u1E2_sG{>T=3%LgO`(NSCicn z8S1lhV`{(Ur)DJM$0vtx3~*i}lln_i0BAg{0T*qYm=k?CaWH_On9*pxUo>-v#eCUYZ?L=~M3;wdAh2omjT5tK@w*B4wW?>Z%VdYr+GK2d z==zLT3D6?oxH95VF_BGzb-G8XMB|F@+~J9XvCD8??433 z^IrJHn1Y;joErA!KoX0a$Kv-u-Ki4A#hDXkD{D4hJN%qpv}h$!n4^(tV)2v|Zrz&f z_OrfZrG~*9WVxTDqpq=r=j_$`{C4`zE%lxjS4 z3y3sukKWM$*lvY;skOuKhEC;zFrf}(Imx#m2#|A+LbGPuTUM|Ja_k-Ajd@``3kz3* zF9m87ew9^3{W$N(a`5}j$PQuNhncrTpoue#-`-EVrw$w8rB^*sTK#EhX^_WI=Yeo9kfXymTZcXst;_zMf!BVlLnJKqS@I5=-T%R><8od^B^^uuOeCGPe z4c(gwNXH19-xg)E_GjEC!azUF9~Fo@{Zj}uUMr3*LliahKY?XG`ZK<=JAWb!qj>i;w?K8z!2YKd3cSGZl@ZE1 zepCNu7F{3B{YKk?c_cL9%Kk56oqi&-3rFTg-zZ{y11_0j+H;=t2T53G)A5-S4=vk6 z4;`Ju!NZ1AJQvrkq1&J0#LRa%XKbfWt-=XnZxY1HU63RGtsk(hkFc~)-@td9bBjPC zBJ3&ZIR^aE3&wy()4XDJX;lo}CkeIk8h8t+V_jLpQz)5L*>4r#v3L&NbBw+0t0;&w z%!$z$4-D4g#Cp^ctR;~~iO0lOXnL|{=& zr50gmkvT&;=001yO1fkVipi=?$&jsRjv^@!{*x*?b;px2DW$>U3L%N>q#Q{RFFrjW zHj?1{zUtkk!%(BbYG@s=W5~fYis^fC_75$}*51Bz(-R(e361oa#(%kjDoi66P$+-^ zb1qtWLy^0_Z^!Bom`w^A7@Q}*IB9H4f0Rj$6t+1*&M<;*600usMDwT`N95$_fwyJG zl*O47VJfWZ!4nolWaT23SOfocnkHC~+UNc$8flL&8f8!Kt8Aa+UFBHj^jHH;?ANxp z7pMLu|99ls#+3&b_rpB;{k#ZFv|EKW#LWm?h3H`=pnyQ_povK?Anl6fAU_ZY)y|ngAl?lw*I3 z4`uK&tffT(+)$T@FZR{uSoO%MIR{)~cV#5bc(fo)5NB(Ef00l2MQ`m;67AQqg^+Fk z7dMvfXEsS?@#SDo7-QE?x0%T^=ecMx_7ms)2PqV;*4mg62?jrL9`je1zD=NIIi~T# zt7vl}N%fiomn@F6c6!9n!ACoZ)db{MMWL^`{T?^7y2i(0)^M(6rfb%#inPU&wOC>z zjEo{2#_;l=f7!k~eEqWik(Aqb%1`%e&;L4Gb3LxXr%+s4#l#VwqGIoOM!kEoBb&0- zMsId>qA#TRb8ZTTcjpr$hijsZz^$d7sd_}kxLXPwI~dqPEr|1}VSH0sw=B0~0t4zs zh70Ic)G?4ufI<&n)z5>2&lh#hfZq_fzr5Wpr_vHTC(`&0$6LIHRT0ih3 z8_+ZwdV>hfV=X)8&5!#WwfA`7ZPDj11B)fdMxH~WQc`>; zXZ2rfz!G^@IZ;(Wzu4G64{{U#L8$!JBYEBHP<;@Yc?b_tg8o9-8K@=bSU%>-Oyq5) z_SH7=Aq#$TPU?3K36)AIn~KWXxuC&P4(lo$)!oPUKdLAtJQgqJV5^vaEW!ZkvYwL1 zzH9)@{9n9<-QecbtRxvgE<6;7uCA`S74{k;rWFsE#F}&pb$xm_GuX&el2Wt>L8~ph zL<}*MInqVnYYE2eSHo+<=dd_2e%eQ z@vP~WWO|QKm&B?n!<%Zqme%2YE4J`dx5vsWcRxuZ5YI4%&(j@|F~P@)TqiHX@O2sq$|g^6SW;m(7122MY{ zSf{<01&kYg8DI%QHztCYI%pZd?2D5-y{Z*v8-W-5lQgP_4W79L4FY?55!`;AkoBY_DkcRDa z8GxK5pVTf;#^mdbhk~{k74bB?gd2SY$8*Zbj}>^4$nnt-+c6~ws2!F-G|vKEtpI^3 z_&UG8-VuswJiq~nMhp=ZqaYQt#k0!tdGkY$Rv2iP6}YlOIwhP?Q)AFXoX4z#VB6>I zoLw7i@IT2m#^Xg2_nS6YE7o&gGPYPXoU#~p+pXNK5S*1i_A_`Fw6yee$OVDvh-s|e ziwX_`1g3xcmTY|tM1@11UJ-lN!2I@@8Qqkvv8ib%QW9=m0$2=*N-Nubyhfsk09`0I zh&ID(2M2HrBNq}K1#`lss#2IitoqYi`qCsUUK75{x*0fRl4prkvy`w+_giOXtRxhv zB_U(=J0E6RJ5ln$W;QS2lsJd!vaE1aW|x@*wwa^CVbE}oy6^klY(=*38AOR$zADAM zdHCcbJx#&-4}cc)I4@+$A@HhNV|8}_&uPf5d#%s~_1D?~#^=!s1$VPl;5oLppG7_r z@6AQ%6m**JB(*Lv9Gxr&8(;OtOzVdm&>1bFkvTCWwaOH!a^rQLOk{UJ2 zsa0)$+KxF;rX5&%N(!70Jcz~Bd=2(@!VC2X(|dVVR>%Mkn?Is3SnWI0CM+x>52X8(7y7io8Dx%UT44Olcaz_ z2fKlCZJ!~F=f+%zs3v0j7FPpv5J{Un#o5n~MCchAK?>#<5^^Z1B7MD&_cprC@@^~d zRjL47ln9AIn1X?9+jE~}&QOtwF2Q@eqOl5+>h^2snNG7?X8g3*Bqe!vz9p_7I^B$F zG~)MGPW=2X%&yDh@Xfm@DA*VZbHELstu-bC_cg$9t9nT}m$tQ&@hK^B_&t!A!(IKc zj!}@mc7lNn_ff9QieJeI3cPh{_^f869@K1?){bKG^$GTscBd5g-1~Y zmVJF+5T3)nw-s_fa`_t!og`#r(Ia{#dRI$T^9<$x7@iBhG`$r+r)F>Hw$Nh=2l+umkf5NK&Pa?WJ z@2XD}MY{Cjy@lWSR}9SOo$N=Rc#QexKAhRTHo3*&_J{TRpMNf$&FX6;LR|4hxqBM1$W~yz>=Um|77CU}yso+q+Y4Cj0^d@PqF)4g}+RYMz{- z>px(D7^;Liak~47Gc%Ed!!$J${*W=GB~aPMg7XL4%fgyE{L^2*ejVMrF#B>XH)WY1 zxtE@J>dMiR8~qtO69Q-5Nh!XVDHKIJ+SvCj5~>_C{k|lz`@@ZZaKfz(Y(C(}0)vWx zi^;)FL=>|@+$D$)L#WFMk~0ANa9)6_AoP-0#AM3^bkpYG>A-Y`Y`4>6tKic~*&VLk z_7?^hnUc=75OB6!W`FU|lt^K;K&qPEudb^7yoIOJof;xfML!hsy;Ogc$S62-`QoL> zuuIS71yRlev7g8FvFjNTpB>K>GwOu=IHz~5F)P(jS8m3@c-LpGcDKey5S|E7dL^x$%++MOU%A}oTa8Uf9&2d=QSeselP?t&UL(9;ZR(Z1nrr30?Bun zQcFq%Ak7&$(pI{ht5>9a!}9TCg82E!Y3?dJ)$RE|e?;asY7FXO|6TO~@dEBap@`{b z&f>{0Z)jlo`Q5*?K-H`cl@0UeXmlnnN4rqzT8vR0aj?8?>5ZxjV@s>DvSVF46J#PZ zE#7&{O1u7pd@a+p_}3FDr9#imj#Xvv&UsY|ypHm!b%_?1Hy^_*v0!h0TX*J;mvd$C z4&`uz&DgWb^7?w_DQRt=*ZwDvc|+t?*@t|udhV%jFDRgSOob{8ORSIKa;m6bx`m(W zFsmvzJ_WEU%Gm={;dok&xxZ2O@neYD@h|<{je~QhMHD5g%C0@jFaDv?9W=V6IH2%& zN$1;|$;%4`)VHwlFOH2I8n~ohLOv!ePe;gn^Q(~Zd||oADhZ2a@dk~GVayAzYsG`{ zYpKv39mx{C9b+-DnR{acB$-H>4y)RLMWAGEPonmKuql$PpkvlY%t;yVc4!? zhMIDTrr?h*Mvi1Wiw(Ztms}k;=9GtS0M4#>P^4E><2W1a*|_2`S^vjxZ(H+wCHK~j zxc$vfMP|*y0z;lsi*}(jEuWo6`#roA#F_V>N`1qHwK*AdNa5AZK`AyW4bQWrT8 zVj{H)uGh4xh7qUlWaldE3ns6V6~AZtVQ{MmFP_VskL}$bBJr0Y#?2L@LsWzz!i!$G zPxck;EZ-Kw8x#7%iCDbx>+E0LjX#-jF(w!^6|V6Nl5VCu6}#x|_^>dUaj|mKFCaT^ zOyTF36B&9}nFAHOhi`^-zI@$%vwjGE-vhR#L#YfcaP+kalZ0AGj;AvnG~j>4`zgOF zjo(vCD0=akqu6}5i~XtaQkTI65^w&)L%Rcao005Iy%{3nqn45AZ(0}j4HYuUn0HBA?WW`=AB8F8oKfZtgX*!IDSM8@+mZgao?6rH7?2?2I{*M<`!&~cG#anV$-GnNwpLa!<*PJ& zg@Q9>_2=a|lDcO=W6iy5@lVk|upZzsL%!MK$rBY@GB7YgJ?wPRZg^CC`~CJ0NtXL} zyi963xU6a5U3^QH8AN#}GA`d4;~0G>P_Mj%GMTE2l2Ih3-@MUcnD{{-9yUVcz- zz|D(xz}~Dqz-+>A9|Z3g$4hz3IiBLJ*OLb$rOid9_)?)Mj0(=!hq*ol*$+bXF@fPh z?zR_4+Kb0=4g9MQCo%GZD<*ZIv6|az*z-$gkipqGdw6qhLlQ|bqP=U(0(Rx3)4CtE zFRw!+8rmd)PSxcbDU$ZQvRloyi~VmeK$9sxJt0Y3;3(q#mIU+dc;jUfMWF8B%xM8# zFYSB8%59Zmy{%)EbP?wF@icb>S(ap|lD0kRZ&@~@tBBAH131{8a&5F=+!-CEZ53AH zmk8V2v!H)BI5~P=%jw*gRaxOTlOF1>q=9;l?)HlNWM?>nwEvtn)>)(KNG%Unb6|P7 zb4^Mr8hKFuFCD?;4m!P|4U`(A5HocKZg(B4`tf_{;|DMcv>>`<1C)(K+gCIBS*C=~ z@P{eNCtS|ZQoZoV*;ej}>Ct*_nEytR$B=--CDw#l=!f3h9|#bng^&!QjgL=Fh0)x7 z_-jL7*!c%9+H>kFWBLS{F;;;;qb&$`dLErTNmyrMpi=!Is@dKW4Bcf-C!Y+^RN;RO z2_SQ}i9V#4_NUdkTYqH-oXt8BV}&_WcJirC$vemTuEP=R+vie~jg(3&iLYBDlILDb z)7}NdSWe3Z^aMOH;7!{9dW_y_oIV{<&p%Tl)}JXG1B032mct7mG~wjGl&bMT?T_qj z&Wn=J8w~E{cFUEuwOqLA2LGu!JiwQa zXQA>5V$6$w1by@wN=tYA@$W0*Lba)ftIUpAO+Jicbay3R7 zv?+$W9DYA;a29BNE-{n#pb)*#5Rva*MiE;qm>zs{e=)Q;*)`91Z_cI`DQ%WcP7 zZr8>RC~C{2l`aRt(`JNBKh>cf%x-mcb=IPcfMO*BPU;sG)nTqPc<=xZ)!W-U1E(a7 zG^dU_;)_m8YZ&l(BElf9))?lW_Xx}bxPdG>n`gD9F|egl!?FcZ&)uGuD1MF3=EXP* z)5sLxkqT3x>qVD=UUmt-YC=~-mX%^Uig0$2z7wEvoB{G2oG(WiVTpJ3_mjcf<9kH> zC!v9mLvYih+I5j*@!O0S3w?Ol*6c^vBdMA0uz3Fdclll;K8CSFeTPf*p>tg5Q!9o% zf6A9TNaQ$OF2Hol;k6$a_TX0n0txu8|5^tQ;%V06X-HNc@vgJwuDMEql(6Vt7uhwR zOMYdD?Y`dtd$y{@nWIEZ_U7) z;O}>r{z|Xs`FAsEoKN@}`f6e8)#c-w;M`weUJOVx^Iye_H8}G`Mqmd8YeEH!mFOKe zFP+M4)@YU(CMHR(RDF6OhasU;3=Sa)v%}r6r*GiJ$_3xsK-m^9NLQu=nz!5W#n@an z1FGWHaFZuui(`&2#GIg5ac+BEZ0Xtx*6KL@Di=u`(Wnsq6Af`9Mzm7$0O|0rj*GT0 ze2=^0HTP5)gHkX0B((o0k%zXC-3i6WAEz78?F4aJ<i zQzF#sFI_dW@E6eyKBJ@Mno2TNV6@T1&g)<0W_v$m@03T@u-62xS7*BA<-7BY6t z+lMVTTJ1@tw(v^s^V4pe9InTd%7V4^%VP*1sx+9=#n7F_tE#T{*)$1TEH();GNtS> zoA{VB**{%rljC(L+x+4xQOa1D9y^J(M%}LYFADYdN1_y#y_jG;f)Y}QFyq5MZ@J&I z>jw=K)+MuEtb!(=LV+7GEC3f!i7Z>d(KYSDc*^zydA`2$Rwv zK43I-X+6{Z=|0%F%dnCm?Yyg2aOS5pO7v(!@RwNw*GnwkC2JZy>(^4vu`W!7A@`|22wg-4`7KuW96FS_t_~iaK``%(;}SSm2wK2bNJv%hh(68uwL3;J%fX!%i}ViO+au zc5zCd{jX*1p3GdYMLP2H&es$&6luOFSoqZQUl{O|xak@!4XW0a$M&!J5fg^$H4t!Z z*#bd8>HleG*)v0nNWaXK<#yt~mBA+8Lv@6RPsHVmOf4QSq8`SjJ!k1PO+N`=g&I+6 z*>b<%YEkO>m%YsoXJk-;FuM%3xg%e`wzTI;_pZ|=s_^mYX%+BQG}sR@lN(Eh5p3K* zO~nA^AuJ~+Bo%(V|1SAft+7gp9@Fi{(DSonqmTDual;6Uv&PE@*DEG{8Yc(>BKJ*YQFna7oZJ zk~?;bHw-Qof^J_~<4X)>!%Ip;?W^w6-jTP}!SNwAWm9oy07GQM2hI+@BH@2OxM>9( zYujJwA0-_^ubhhGpWn|8wKbs8{C)g?8mY^HH+T*pSP=aB0yhT^=b?e4V_tS|L9ZI~ zIO&IU?<%YoG4K3!A>F+O`^lP&{QJha0V0G&v@( z{CKU($inp;dp;~Lg>1}CbohHP7z$_RxFx@0iX#M_-P7-DlY7Yfx;!t6vvW56AveM_ zU(B{#Ip1URk)~H$hp+X3T1{*UD+hK>Latkk8B;?kWQWiP+7AUSv!#KrK3>4yH(OVs|ew$E>P40tYtUe@=839dr%*|BI1rAiF~krSF%dh=8OD z0ly+x86jUy2xBe(^xB#rSgk3;&!dlSK8nu_v-m7_tKs0vN?#ecI^i;*c%&A{?zYKT z2P7bY%LH@`mo^~sCa(0cWnTi}D!^CF+(VNAqj!8n|B2Q1s019B1bl0d3; z!871Blz=@S=|BUnK1Dsse{|ul^MBzK2qdS2xQ&QoXwbpo*Fu7*fa4MP<8QwVw;c^x z#j*OVTtjk+xesf>mabhG`bZ>7r$B2KlFpXsiB)kV76YMK26CW0>>tzg_uJ9G3v2)8#JMYbh4wCiV0tUd-nLz+3Uk_$)nM#8-J#u5c@W3c z^6h^GLG(5C-B2DH_zXX|FO&^I+8DqkxO*!MGX70&-PsX`hOFFr%Qp+=cU?}~8Nwmf znN!&bPG=~fQbR{9fCC12V>mFu8xb#iGX~v#4L#B4B0HJUL{zX0smj@XTo&DNeF1FG zaKyl0-`WyI5EoiZ)g})156CKAB2i>cmyPthK5RLr#T$0$p*Q$%HL6YNU~w}N_pYN; zdzoUjF6&mUok1Zb)IH%}6}wy=@sxJPc0iFx@YhDjxSDKKb8->glNB!wPbOi~IJg=H z@#p7m9>LbE1bZsv&LLJ0CVFlJup^Q&NTnN|#w%3STdI0k7vjbHZ?y{-9mNyVnzxcp zmABNwo0oHI{ULd20eB1+`@h!LkmPAq72)-l?c?Z=^s;=_yeliE*=T>NAcUju1Vde7 zPq$x|v%f#9nePTc#n;<$t6f`5S*@|>{ET!70%?}T)Qai)<#R$Z&j#i$4 z3Q+a@MJgi@i#xx6wCW)1G9K#yLI^$!K+#HA6eI11Co@6|OirdR<_t_sFv4z(MAH2a zo(|!+0z-vR0=kUoUV4#_IP=%eT(RiX$Tyrw{x$=D)tw0dEa*Lv9hRPRQQS~%a#}tD znZGC!5!K8PAR~N^4#YoQeTCO1@B7GXJy7^c#SobvPxXbe59dj0>>6VZ5#X~)-E4_w9op&3}tXolGVNd#Z53g)VLJvRx={lUah^+UkeO&NH>@= z`3@3>hS?6h7m!;K^g;+l7Op(vOuH}ZOBHP{CjKwWmlt(s z#;Pgl&#k=G%1f5v-NC3FogtI-c&=;0nWTo8;BzDNVA!g0%G>p-0Cpl>)<uw0)%B0pB_)0Ct*viV)ESsf2M)VE z*SO*Yu`LY^jmWr%C3B0;Gqu)DuZwyb@7ynhP=v+J znI^U0dWL0waj4b(^GCC?$0`((g{Stii)~WRaX9xq`dD+ey5=lPjdjO6bfy<;Ca=EG zqmc8RM*;@nt$(h|1D2TiKsqr#0F4p;Nr{j*L^8SZp@9tRkaJ2sLkma_g}ruRJgDpz`Rq-ZAn z9sLb!;J#(rpZws$)_TXuG4nygPpc!xO?fGaQ=Hh@L2H!cUViUoscO8bIYyifFlwdM?iZ~#^fj8 zLK~b#owX;z==;aa_3d`dg=LbKnGm3$*BLxlNB9~#Yd=f13POZQa_~TCaN5P5k`Gxn zuYdDCWgz64I-%IEWOAOp#pZ6ugtRpKZx@-dr)orZY|4 zwDtxJwEvEno;(vz+nZYVk1)_EeGA~T!AB9_)DC|{=;VPbNeX7|=aO(cScDLeSxpA(9xPooT$%*Vg%ssePX zo9Yw_*uOL6wJb^YxAh6O&^?HrU4VCXE~V!&@S;!_2ueV}{;L2=mI2rd;VHv@h!idE z{Ybj2WK#ni@xZ1zxV^x9hm0$u*wW=QgTI$C?@dp*htMRegecOITjObFEk&glw9bZQ zPZ>4MO|(#zkk3@hB}vx|-u!BDopCJSusVQ>1-jWO!`VsNn&Fu7-%P_sE+hk$&hn)a zW5<(k9>nTqhKB;8dZ^KqPpf3K$V&IUaKN_sOWb;NMs#S-t%8xLwPTNIFICs$UB3V! z2RxVvJvaL#I{JAlMuA`?4mK^R`A`!)_vC)D=s$*e_n&Spyg5H;SRc4^9FyxMQsJ|! z4l1G3vH-oFu_O6#*q=|RKwb|5+_O{i9^eukvnKV8lB+Q>_A45X?Aj5Na`ghf#~Eq$CWFMh0rNxTJ1h#2$s?efqf_ z6u%7q3nJQqb@3SwCVZ*e_Z)u_NN%^v&xJuoK6_tNZ(O;tLA2bn(olf!3z3DD6_K>G zbV#j`pWMXcphp^hzV=hc{q|in3_vxc(EOyylBWf2!^(J5k0DV*kJi3!X8!rUR-} zpuKx4~@2KW#Q58>rkg^IEJ zAw5Bj>*U4L>)2NzpCl^R#HN>qx=OH*;J$-7ul<&qZFRpHFPnRWUhDYsbwet3L#zgw zTTCH6MpBNmW8y@#LivWby-M)<@JW0JzB#-m;WKSYRQ;>Y=g~C3V~}mhyv+%YvCE8Q znAKd&s#|NH!NrYOW0?|~GXfhoR_t@PlQhR zQ}#p_IzMm8E!(qN(g%KEm$Y<{1G&w+ft99nqnD@m_2h89Vc2geF?sNx1fkK_hHP+1 z_2v^-`JY^!1XyFdIaAViYczPEFac@0GO?Z2N!s>1=jawTj`B{k`hwEtt3OhKqr#e0 zAZfygC>pRWVg2$kV%PvJx$}B2eQz@SxVLz!sT%CSIjV#+rZ{U|jcK^sNKXi5b=Lm} zcW)gQ)!Og>58W_?bPe4lCGCK;f^;{Ev~-uGbfX9oB8rj{(lLa9l+qm%N_WHW9`||9 z^L+n2*ZKW*-21xrh3uJGv(~-t^@;cA^}g{FptopKP3(XMP(Q8*WZi{qES0R7IJ+C% zZOlCX0n;AJ`y6Mp2tsxqnq*L2)lo zwjq(XAI_FaEIqmFAu$4T5_+Woexg&GsV#hyty+}SzuM`p1w{vJX22>J&ILjQGhlxT z149^P=HPfgQ-1#yZBSpXi@oaEI#|C?_T5L3&uoL)!w)Nnn$6JxeLAWCVa7=^uJH)H zIQWt>4|l~M9i5ozkRCelC#I!d`WjyNaTX2#_9&3o{yqJW^SI3ny1b70BCfb5(rE^i z7K|K#LS2Uq2ekMcJZ8L(EX=r4Nu1MUX@}I;EW6PcSoY~S7Ir*&)$#O%@DOR2f%!jQ zjzbpI5qEzU97^4(wRk4`+WU94NfRUX@U~Z>7_K265tgXRM)EJW`VDtffZ|1RcmV|% z*m3|)PYXmHuTbf2$LQ~gUnEWIc>(LB!eZ-|tUt2+mUZK3?=jcGK0D{{tx95VkppPg z&6>-Nq6cXMz?lK+8xfc_O@7<4lzAIvz>dH7z6YcU?g9-_f|+ z5^Gh7eV4o^bp3`#3-DiG%b9%&8yE_X7~ZzBvO)!e8NsZz?c+v;njDM3jVx^U(r%TZ zXXj`LY9xda`Y~{eyCpHeA16}J=XX)2>z`51O5{464?Hymjc4X`Rf4T&gAB)Pf*A^H z-wat!hkEbCmy$@F@-tNcK5nw06K8BKQlkA1LRF+zN>de2A2N(vERMx z9vSoN=AKQ06qYhw+!5SB%{{3hd9L*7Y3SkcFol;A<|?He|7 z{CNq9mb$kGOn1Oi0bC{I?C)PO+8ijAf=U%61mx>WQ%~RCq3cIpAP^{n3V`f+uZHVd zoXPc^eGRf;2oM6WoJT7lm+S#RJA)>$Z<;iE5wz@D+uP#;9rf#t`Dgz{=dv_Xek2?D zeGAX(xZ=RkW1_r+ySy!$HR+Xf)URL9D=7xUyfC(li1M5GOzyszVU~?@`4+h5Jj%c5 zu^E^Z*&T|8^;f(s4J}CP0B}5@fdl+8aKuwA=b|Ee^UX4nlCkT#{*N5ZLHQObH9I~& zwk!7sShY&|Q4qZ#Cz?U>1T@LWX5-`q(UHvQ36g$!C~Zv6I6?L~{(ktd3Emy&K*2=C8We=E>z|L>Sy>sEky?~oCbvft4W)daN3pAl&F=Gv zv62sc`kep8j|z+gQ%Z(^--;DTb$hCwldk6Ky$gh5IlopWRb_c4dKg?x+rnF80GjLR z+dw^!YtWuUqK!c37D&A|p6dQi{Lr!v9cFeKUfJKU;L0Aba&~@HJq)N2o7b1`BSz2g z6$x`PjN43T2IA-I`Ueb9&tI`CtQhW-S@rUuHb#KF0Td`=WO~VCyMX4{VusMFx0k0l z?O$2|e8nfZAykT8d6Ws+9aj0!EIAE4Fl*m$MT~0*@X^N$zQueJP?0q<|e1 zQl_R1ftyU@oIl>60B>i{lcYB?DA>{??@=kt0`JP*WI`83Olss0X9#j~C|9v!l#zJRIf_FGe@L=ABS1QQe$+Wehp70RJdf{w9UN@Ir2%bv7NDDgm!8juVBJ1l!87~H-k;dBEuh&n}Xr3!Ed+ zQU}jM&<90|Dmuew<*7gfbioUr(@5%_LFtDYMJW2nLdLiFUlqHVJR;e898qtIM)^q$2h7qYG=ai${?&4>!1G3ze_k>^mhegi+Fh~2p+JpRGbzV8U zK{Fo&`bc-4nOZ>o1K`8X&hhM6PiT z*`DcVhqUAl88O@GFOGq#=KJAQH)1cEkpKX)1Ccq{2}<=|?4BNEu80HC&mB{LuJpcn z1JwIr>Jg%#T^(~^Y;2uxDQx-PJlOHL0 z%s(pyUI~T1Jy%zkKrUr#_?R%R>sPHVD7yh1AQ%uFv$u3`zynH#>KlOTQso3+8UO)C zz%VlfsO%Xz01hsAE&?yIx*6~S5ToMD@6#G^rT^`MDgZSLpb^bUiex{uE48$X$kPs8 zn2gl9|2crBZ+l_^b~VoT{p+j&=C+W|A_|0u&SI3GQ2yS=fO(W2)3eAuuN*Wba+nJ`=iUWbwAZEy)>vN58(;21}Ly&;fN=$7vwobpZ9=%kh8QUcewK@XrEk@2?J>l7{Am zB_-(Lb*#NN86-eJYn`k?e5}g(w-9p3fsgo*ulG@B4@_mnCfK$X zn?E!Esa4uz$2_b#$W*`tvU+?%fiu^o{cs8p9*NL!jnKgT4F;OZ7G-o~JvKlnn3ooY z;Wzqx)W{luS61B&J#K%vq;M1=9c93Tcim80RcG7r2Vx`AB#Sje(N<6G zAGEeatX)Zl!1VdUACYPTK?D}SGarpBtoyJzD)`s%w_cF|!Zy{nrb8)>qfohv6=W-N z6G1rP3FHM>#zMh%Jm>EYPPMPOi!ma`+scevRqlQ88JWR3croR# zHO??PT{z-gR~TmUS_^l~-ZR7@F)B~`>(%F3(X<)*$>x-i;~O;|**@Qi{vpQ{h#A^!uY6$!U*@fld+^A+RX^Xw=}&T-uKGzVrtlu*0Z|cIFrcG=S=hGa^D?3Zx>ZF+ggj)k`5=Y7$B2HV3j`e4$}4Ln-6YWiHWo6*vJ{LnwfP_<9XgwJra?~Vf;GH5{} zD+J_2wN7Ab1R`G0o%^f!ECKRh>!-yNl!DecHt(NWX2CV`>ZlMgYPY$z?gI zc7vjWO%BW#0|*b8i3G8o+iB6drL{F0vL_r0g2T#*-AgP?@Z18E_ZfrCB7TraR`2aM zekeh8IC6+b1B}t@`4MalDDmn~?_{_Y7VG_%q^HgLOP4v!+-v}J1@PF5yrbl3!+@{0 zqwBfq1+Dgr{{`Y^z)uAg1<-$&!0%mk%gV}v_M2ZLcFdAp7&EfNUw!XyzJmJf(MX2u ztd@fiYow<5Ru16ZL2?ALC@T%w6C%P3m;COkQ~_;YWtoJHiMo8m{Lj|?N-9dhdW}{& z?z5WG`I)%reA^u*yKkk^&z+cy`vi!=MD&PQ!i|nQFLJOP35d}U6`@A9O2FAuXM2G> ztckd5L^lYIk-x3{tiuIDI#6B!%}Yu;x;wFBvRhuXSYUl1X-TD2bUiWiP;ln{!hlTx z`R?(dBw!1UFMvs3y-&xly&~)Df_eil6fXI$^eRtBtDeDRys!m$+kHf@n#G<2*{Eh)kz?kxoUCvwr^z zsG#Iq7Ld|JIrVLO^P*GkdlUcU15}eh|JWO!3Ke)FOJ+ap)fOG(9ebk1N%3vAU4EMRK{psO z-}eR#1(3H}?e8qKCYs+~On?%!a)P?`$=EHnE_-#LU~5I=S1KYZCbpjlz>k`$mcNFd z(TZP6d=CF^8HX$7XdXyN5fAIKrak5 zzQ}(O`Y5HCd!}aaO0Zu3{hY0^%dI5-9V7-E2|QE48v}#=CSP4q zKwH(>nUKF}FKkkA#?urj0YEy8S#~Ae^CSF-=5AHqjkQhk$NUDfZ<`Tl6_B8k2Vzkm zGWuKOE+Rt`!t~7pMM{`iM30eAEbs#c0b}4V9xbf5Wd4kD=Y)SIg5&dQ@PH0}r}8b6 zjGLSzPx{1zS3F_DUb*6CQX!fbKUKSO`UOA`@1@m`ik5*8^m?5ZSLh z_P!-e1`R`SHGvXX>hF|wZU87fgZvEiCI1$;P9GtW;XpnNN}dG{?03L^X42q+1NdhG z)+#oMZoXyJk08)Rm{C`DF{aE`s9=sbR9pxfEUZqT{{!6!Na+QZ0^ zb`WNQh1qm6?LbMB`l%7fZjkH}P-H~{S&C(^KZ5ebKiXm?hCuWpvJ-~ssGhfhwEOR7 z7g*{KM(*gm)CEC;`}*M0#z>)+^>0x30&g&CFJj|HC3W7BN|~MJ56!E1+*!)+++Jha z*^|g9dwO@IRpp1#6=d*4s#*^S%1NfIWr=fE;%QnVjKRZ7-IDc?luSFZfBPdsG}rlt z(E3IcS;9cObEKs6w}CHp_O9+(SMs-q(c~9;LTG51_&*h-Boh>z;eZA=U9tRe!#g1{ z(K^7SjC9_C1_mWiRjqSC#IYuU*Ndur))D2>Q&LhY-1Sns(|Ijl zhqawu<~fQ5CD+t$EA|IcPsZLmiGsH~G3Rf`K7^ z4dt!QP0+y+Fg-iVcD;@-wlt{tQCRWA4Wl*v`}Xzz_Ppgb_?=zqyIX~?mXrYPhcrZU zN>R|rqrv=cL$2qizO{`-_f_Ege2p3l2@*L&Z~@vX*WSSF>hBAGZ5E#sfW`0`B-g2( z(X1AoBoKVc_dsV&k%QH|Io`-0z>G0N8DXv3^;7wG`2thi!Cj&)`V#4o`KQ_M+@1L{ zW8E}`UUi4t9}$CTe=V_>wpt^BcHODXQk;`4zgHLM z>qDtDV#nLBr)eN*#8FQ19ksufISoc=X0AQcgRXqiWBy$j_`f4B{$Hi`|2t7NQrTP^ z&=}f1tsi0;-Q{X8IXo;!nP+SyAGpn3Y0&4sD5?g5On3sXj}QU;uH`$p7}Yb|QOf|k zTImCn5#Z?t5k@VJ@9su+@K`eRU|W4pTujnCfVxOgQPJYHP~enTBY*;5Xg2vZ^ND7F zI&tng3esx`>EVZ*ZoU)y0I3QFE$9#Bb0O*9zL_+(y)c~%8^;Y%EORHCj=PT_iFwLJ z!Re0`Ij&x^Knf&sw>N>6=a|5N#*_aoM8b7(iw~ii7~OSfpFPYKQYp5M9I(8&eA#7{ zrUAgQ08z&Wm|mtJE!ue~M^Ijif_NEIT54HG2t>BXdPr3jQ^N^pv~Dz?n*y_Npi^cc z!AUl_548aO#+U256X5<7P=9q*6hP7U3}txO5>i+7a@eVNjI#xeg8`ECSY!)tw?ohE z&{@X`@dG9G&n=w54ICrLz$z(0!5FlI`dWb&8tfES`(9pNVB`a!62N37=te&10jyrd zbL6BGXj}0B31<8s7eGA(iWek`%mpa#$~I5zzdphOQqdQ%uijv)7CB!jzqPfs2)?Hk zuxGhnsyFlLI`C_e|FA0GQ}i6Jw25-*fUiL3-1%!KJx>?aqx@#@vT<^q2PSN@{L093 zb_Tu8?@V&nwz?aXAAvDttbzL4*ZxB(fNkcO2WlkXsN`M>1V6rgo~A&I_`u)%qi1O_ zp%zy<^6Y>>u5uF?bkJ{mW*@V8jpIeL+mxA7a(K+lacDR@IX$R+F!(({)(8zGYq&HR z9y4yo@Mpzw0vo1{d8VP-CgGeO&cm^f|K!lv#pmgUX`7JcEacA^x32VcS2r?IbAOaj zE+Mr03SzT`7J)Kn>yp{lVcCt!&Hx#x9>1y@|$;qtoH%M_nXLu zG5aMzMInq9Q--g4CO7(=xkk(M#FE95m ztf%aR|0cSolBFV`VGlhL&6zupsO4)#K?roM-M6R9dU+Yei?Q&Y!@=X>EVF zbo#~^EMcIz;nIgq|9CN}MH+Y@{hwED+&)uo~{#uA>+!c zqM9(bY5$2qrS+Q!Q@(j)w>|SYN_H|mK)Hd3UCLsaak>Jq8INqWD!Cno1)KjkjHzg6*eZ6jT=b$_$bpNvs0 z2WC!Z#*=smu3Y|j-K=CVnP#UJMhYWJ%7@+cXjILaPpAH;bEosy&HgT|?RUmMQF@q1 zP8XxsLM+-MP!iMs@Pc%%>3H1u(r+G^?{RzXTwO`EfEcnvATDsu1+gUXE)AqiA6ODj zw_jTY&H-H^fE)-&NHmmh)>bxwqB+v&@shfq5gYWq0d)=t^fFtHiUQ+TyB|uaassi{ z*5$qfbH?;((%S&MuBq5dpQ{6I&pfw7gT^EMOKP>!067P22+l3$qtDyx2g2&BRq$q| zRX3mpL=Kk|{hH+k{96#p0+ROBgrCF88W0eMgDBL~JhfEyV_(fW*tkHyEhA)kU?(-7b!tuX!?}0OrY5Wt)_v1ADP`{_)~K1! zPq=IA_UWedcG2iBzVA6bZ79%jmc3gOyVD_Q$8S1aXRES}(8ACRv`h)#x#}>)#)jpk z6Y?AO(2ZT$Do zi{!sLW3@`5^XV+^+=|YeZ9`7cAlR^A=p8)niJ1@XV$?5Bzl>NH4ju=#x>uJOi}!hU*J>pvS178(xaeok z>q6iPwuA*%OXqv0&MAy%4F6;5?;rExy-4)M~< z301qrtf67~IF~dpw=kfI_WV%9shwWGPS_^%j8Y+KUU_D7zVD1ih=c#g@))aT0 ztWrf*?xtHp(bLQ7FMXBTyHOr+Vw4_-_N2`0?9|P5&xu3{X3!LjY_t$ExJ`O`c`XB{ zB*}_Zr=8a=3$|{D2c0OwjCQ5m>`Pe9O)uhD>*(s{w*LSc1-}MHp4${PJ-wp5JR$?{ zwTLCZ3F$;x5)sQXlmlRp1%LaQdZxFhoNQV^je(A+0pUy?4Dc{O)86rl993TevC#y zSSTZtUmKX^#s!!oE5>NMTfRqugsnBtEpJA(H3Xpw3PjO>Qy6fy!tfONTOg%}pXK+k z;LK%!GJpwRkMk1+4@Q9K8k1}{z;Y^*CMwYhn_Zt*S5ScDsj=ow4=Z>+7TI*upk76R zR|!BzZnVv8bK4|7d$@3FQkRu4UO3jX6rg=bsd*CU{!LH?Zj2ZHY-giZkArOQqedWZ zPa=ctFrSlx&fDs7Aj0t*i?QPEFBwvZNoqS$IjxSoyuq-E2(dXeXOf7HMZi(!Yq(N> zs;;;Wm}~Lk6rftzl3}Z=r$9&AUESq^KKjw=J}pLQ?U*cei7=QqR)j@>wp$o_ zk>0J-fX+Ypnu8cMsp-|BrbSA4t(}8|JX7+d5bWLVrO#@*{vI%I6_Nwr7ns;p!AN5A znwtX`)eYXGKoLx7A!(x1uvT5v-?RS!w}1vs100Bd`jK%o&l8K z4m@ZI9!cz7Gefv2?1XtiDK(K~$_n_(m7ZT|ii8!{V$~C&BiEl&P{JK!45Gz`XA zB11$0)s8kYGE(5~-7mIGs-+@V_6Tk$Ft`7{sjyi|j0 z@Q;eZX<~&_78)9KWMj#YlN-I$D0KN?u=kHhlp$lm<+q9uA4$edZj~SN7KNP7mavuW z4=IM+q$7cik0|8SX5nf-o(K>kPHeF4kWA34jl>%c2}wyRd?r6eY}oL~NGQY25`$uJ zaBwpw%3$GB3nB&#@mp+SX#sr_lJQ087AZUIyS&G4dXn4<6lxENa)lLO6*~)sL^8|0 ztKa$ael0xy?zP|~+olbZpR2IK!#BSZVUAyXuLT5Tj={x$X_=Qj(Le&6ppvwZ)6i3n zUOh=-3JO?A2oyg4V(Qx~mLLF3r>!*j9JINviLL04H+mqX!>1p3V8c{^#ss&VL?{to z&+on%8w>op#)1O5i*7*NW{QSNF}ziyq;UG>ky}WlS>868h0ecDly2VAyc+W*ciRD_rgCN~t78Up!3!323&jv0tfVbYA}oU*>DiZM9b;3dWdr0D zuTP2hq_Uz3vz%0`*?6UX&^>rsNbu7l?AI`@+bfp*IUs~a;<(-JVZMka)L|jHucn4a z+hZwE`;)pvGh}c*B*a^cbOa@Zgo!!=KZ?kFH`(-$2|8=r)0A-Vrb(;0PLW*(j>{O- z^;QsBiyUe@A@8lWt|9-9-xV8WNPLEBkz~TS< z&;Rr={*QcgN-tFEY~FXB4(WB)vU`{E-zU(st=0)SCU58DWGiq&oXSEJDtrA?whPp2 zo`DJwSSjRgQ~-LZZdWRT*s}lz+MHc-|Gr>j#0UY#a0^D{f&BXz6?8JiXUYHTs?G^6 zoaU#6K~Vu2U$r+6ymmw$Je>b)F*R5t;fP3oKuwNt^IPAK3%Jhm^ttx2E!*Y?sd zgbtYzO?r86!|vsBGIPAR^l+R{=M&dIaay=8&Bx?2x93BJ`S~bX^XDADjR5Nr7XD$= zW=4_rgam=mdv8Dv{t=s2q6L4pTG zz!IK05+MocdA@UY7Zk+-W;vyo#j!Dn)Cq`5TXuo&Z_Mch;Q0ZZEx7CUeVUy6!!{d` z#`oIVr0QRSvf=Lav_P~741+ek-h%8Luz+fvI;|>a5)i=N1HI?@9x%|{lk-#-RB@Zs zz^!7vTG#K{I~l|H^^%@PyFidi@u@c1_QS`WOrk79b^jE(cJK)Deq}^G$TjV=dR`P_ zGJ=p~j#VyC->WQ6SoCpW#(zQKd@P3z#yyZTT~}K}H$2qT)ZiObP4NIm$nexuj53Q> zUFDm%Z0doYI-9q@i_S5E&0*iEw!Sh1B?Gb}Z022LMKc zi5LVJy;EQ!uCA`0^`Zs#jw6$k2B*!*gm>9g2rEt65t*&hF=G5fg}*SA6`bDCJGM_P z=~cgD5_kvcyZx`5IIv*iHj~@$t5j}xcILXEuQfsHp~Klwc>;{Qc6cUtvBo$D3U32) ztd-|1Br%df!A!|=fLlIC^pP~N=Uyj?rb_jgJ1t2JI*5%oTX;ZG=IG=Eq>>E@@@!6mr1ct)1E)G+`tFQ42uoUrahLR7WCVrq0AbJjsk|30D1DO1`ki$VG}qXe3=Mf zEtywSHbV&}2&${YYiep563}$--CAK2G8Bn)uL)6R^E03)o(M0G%H4HL{qIlCfufr7 z3cit^eyRKoUc0#T<4mZ^H?ck6_yRd$ zgm#bh+WMG5`qRWm7Q)3FZAt%Lc4^^osQCOu_`l`{7SD$}Ec9$UR;d zbH*;@q>4%?IV`}+#SY6WW8W0TM+4L;W9smqzC>?j zgVD5bulB6Bl_PgFelDK5p#&f8NMU9(pXDpfoOl}MYhqttUt>qWbz?Wr>%nfbj4H&+ zVA?A>*?p5aHKmT@E7q3WEf9xh_k|Rm2R6fml~KXplL0z+mJcx}~W*%?ryOT)dh@6%OIHKJ;NK zO14)=S<9)zGco64+{JJWjBtoOH)dCJJks3mFi7N9obxY)jnmqUSb z`qh!+_ScQyE>n0&;Bu;lDrC*W&n@ekFJG#h;i z-Kv>1m50s{v0+NXz7+NME;;>gxDxr4w}G^I<+gN z{b0Y^37+2t7R+E~9}vFiw}vH*1RIZGZ>xj_O8Dw2sz+>J+(C$lql`u8E9pM(aW7-2{IkVQX0;nZ|Ri{rVYc|?|h zN8g=^Cq6|#Qzs5~Af!ebwER5&N_Z{*eJA#cGZ*AsMH*3i!FzkFWb4&SQ!`7X4nD&%D5adWSi-`m+Ro(bnqNxwqkfcM^(KHfxKI7+o6R2+8`BJiO0&|;5?%) zdeDvUky3|*e{j_e&eebHszod5eu<4q{0ju@%@WL^76S08=u*6 znDTiUWise*CO%b_5h{Y<-J{SJGf^+iVAKrShL3ltzn4l|GU9Bb;ODjCG0nYIL9>f7 z@;OmPgO4|>R6m1Q|3)Qu=flnID5AM1<<3&G*)Y$o$_WI<+_CqXq#D8o!(S*#E(@%- z4-GAGM(DHa?O$|qk-+ep36{|M87^898L@q!`Mg;y`SRrf>~T*RcT{-la)b~3wsSk( zE_DyjUq6+iv~4uu>*SrkU35uAJbsEhU#jAfm>v+}l2C3TY)W5+y)k_d0EvmU z>v}-cR*h#tOip$YM^Zb%#Xa2?R^_nM{^8O2(?h~}We2hMm!cRWrpui-Uu$KdU520TLwmxpMzOC7IYN0tkdSMi0o?dGeAxfy(~PlD^D;60UCwtG3> z6@D6u%Jw9GFtObo|7(sZrAW3wqZPu>fu8BL(m2+*stlx!K)B^z?Mpm&LuDA$F1~_O zaF#aIno3huzC75xsDI&cDZ8Rh4NI?In9Yi`5n9>0CGX~9O$W)7iGa^!*CpnrzYlB< z3O7|qAj;wDCK?n;uIdJnAx_GZ@561e#Uq=}CoeSmlPU({7cc^f1e=Y*XgHK%nn$Qj zGLhe+JkcLilOQ3SULKToaU+I<<6_f)X~2sqKtL30NF@ESnO!C0 zfL@XOP<))axiEi5=mA6FZ&cNmjxBo}U;fEaZKt|r-Gq@jeA4bi;*v-e5{aWSvmgfU z{(k&U>BV~1uVr{JbjP01XGZvaWDalCm>x_M*cPnmC6Lqk>E z{s?iJs`D`^< ze$kYMYAPLQx4ZGVfgYHumX$3|ud0c(%5#u)v<*M*T;6}}+4cR1<;l}0_GM44wMj3q zq8tdBlZRFXysoJM7h~XfW0OHa&0!noE``f?Yd0&}7W3;cArc&G*E8{sWaW6bqUYOv zte<&`AM-wCfe6>tp)C?ae9#Gd5KU2V;h3qK%Fp{1e?TM4&M+lFNF=aw6?g+m(r{6^+Sa12)lJJg@t zVbTW7$i@;2aTCzg51gYvk+E3+5^>o-woc6Fkuc1+usVlP)7R?Jw#c z9-f6?J{3RV(a-ovNmsA;#W4zD2|fqj)a?gpn>;4wnawIc4{%q*^Siik7#9IH2>Sf3 ziEtV7LwJydD|Y+!#hx#yX5Q1`}>gS1#+1QPRKn1sRx1Cv89f;^hH3LZO~?8+r_}AcP)1 z%TpXm-|kt-dCTt=IB=qWRx&Gzt|^yay;9TiBg@#muIZHf{AiA#^&QGr>V`wgYM$Vd zV%D(uQ4{BNL9tzoZ~n%KmfD|+`g$x2CzMxDn*~Bl_7fJ;IWK%{JutLzl~E|+a=0j* zVnIC@3#i@QyV<$=M$GG&lj2Um_{JEQt7pbz9znr3J1K-eZ=JJ$JkL!s=!dGPTFIKs z#wAE9u}hdrpVP-~s~@-r4=CEd2_SzP z=uI!{R*S~4p(3Xzseixz>BQRTViAq+XR-i0NMS#QdO7*X4AiWgxbHB{kv|H9zfHfa zNU+7E^YpF#+)n{pIJU5e$k;XdiO%Xp1S}6RH(4wwD;5iu1?Ea_C+Q~X11qJ|C)38~ zvjKVUn>o2?CJyQLDo2iG5qYy{mdQ%6YKYQKQ;O5(hJs75tfOHw$~aNL_n`p`FTawt z^`2CmYH#leb-UGrd+LwkU~l_%1LvQ^#f8MNzbOn{J3MA+jSbBPi{#E2Nrd;)c<}Y9VJpIgD&X{TZ>6#AGxs2m+dOT)Agb`nK#ME78 zUP{%^xaVz3G~q6I7g*OV!;kwPAmp^wxQ(V~9L#CUO0BG}Bxbcf{yFT8B=ft(A};g1 zUn&eX3#%vgC4cA+;n>RZ1&wv(fBb&A-}_ob!q%&kD8kO|G(WbVjyOose*-AT-u@l}(n0|5OHn&{rmSR&Z*WbQoD^kb{ zz{e+;D02v)&?c?vuO-E09vcmRGG$gXXl85}y4e8qoiR3u_CW@@qa>ksZJO@r41eJ!+6*+u5*tpp96wQ?4|FdqKSHZ<0+{=HbelZnFqv+-Z{tSC$o9H%CNtd8;>f?~IPbuumUKf44mcsKmRV3Zo}Nxb1>o zxm*S?tPaso>bd*~>kiy|t{pYITEeuAS2xt&(KHDdl4OdSg@QBBy>1DIsh@m{Bqh{$ zQYMFdu0UJbI-$bcN_wFwu5+yQP52>Yg081D=*x+^VGL|Lxx({bZ(2sd8_Tiqvj%r6 ztKp)HSs{l0{*#T4L2P)?5VEs!EWhr$rTI3B$~~~r82sxk=E@vQ4VY>pNDsB;pl9iO>32WrMFD2IqC4c z#L;`o=d-rM;bIeo<2Y9rC!f<IR@`H8)g3t5)mwp+pqVOeyRVSlue|u9t{d}lRe6GPGj%xovtU|3?2UZoH z9?r}d>UC~!?b?vb4`B9Lo`?P|#z6*vEi1*ct! z`(=f*`+VqUQGxPL=3-*FL^1KO+KP|8GMbuu`JxZwN?dMG3FPbzech}#diyTLR`E*+ z!as=Rbj}5sIU0>$q*^&t75sU+b>K6w>g+$1VLQL(@7(+*C($KlCE=8I&>>fqKSnid zac!3-vER7)NwZjl$*rkFgx1GsS8;sp`7Kt#y0wB|yHfnZMRK!(jEr2;f&oQ~-0?5I z&H02z0t0cEb*X#C43k$^io4LFx{Qp0&59(?%CyF*><@xg`Ml27U+;xQJ26_(U}snT z+0z5JhCW4~SdD2zGQ#NDW|E<*1DRCB{wk_s2JY8fKApk4NAClJtT*c7x2~)OGR%Rac%urlNIBsy>4AVA|d~I#-M$$zdJ2WBJ?1(`cmy;Fk3zpg^R+F1) zvPHD*htJ>K8^`HcUUJf4Uuo$Oo2TlsuZ=d43jLW#ln<+Q_n~Abn-)>|BTg6zyxVu> zwqa2u1O$z_Nut6cOW#v9m_7)iCkV3?>`%)(;lPF^1)|r+YhM#o}h|+E^BbN9af}JRayC@o&=s()J&JSTP3aDKV9QEQybsD(&NBziw=$N@`|^W zdevE&DErvmhy59)ot@+C@!f`tizM|T`wS;`S}yhl2PZ<#=gXB}MEo`6(xS#b`I?ZR z?nh!uos4P^ix}a)vrOyvik~x~CYcKiR2CiO^UrYF^>f6J=n%izC={8H$9`Oc_3g>W zwI^=^|9pIHJ5&AH^z?BN*+pdJ@cfvmKSP|N17VG>St9gNj(^h={-gt$X@f6i#PGcO zQiAo|p!ZK+34EV(aTZi31*qHn=ElY~QKBeBRn9a%9$u+@a0WM%(}o+=DG>_DdM z-do0zoTM0*j7AOXqU56Dl;@1*&Ky za^64M*)g-1J`)k>Oc}@W&3d=ShZ%)e)ru|RQO1ym`#+L1Sc-pf`QD~yVEo67N}641 z|DhQNM)M76Xqa9$Jsz1}X6915sxm7HR8{`d;@W1I#q}{t^$)QH&<>_ zrkbY(m-Nxnr|+^sY*B;y!bH(U*;TkypahiX5X}%Xmyv}l?efIb>htxJ38(WWr~Rpx zrUP#t*^^vn#Meh~r|e`!xTW&D=Q=!dg(QZJg#7KDGbMC*lfErB%Atw)VgCfjDZw``n&>ZrVaQ00 zM$hX|wkxKv4DY3dwa)+S#v_}R)AgJG8fp}*jyN8>SZFw)=GWx7SnaW=wfAOY5WB9l zo%yq^v1#laHka@@TVoxha>37iaX@~}Xr-bsLCX5PC)|L+`MY4hyk&e5oUepdg0@)GR1**l( z=?8(WBKQfEjwwW`q3_bb{*eErE>k6?*^G?u39M5X5^uv3*K$GzCtUBJicQ&E88{tK z&6%1W30>3ql_1+A@KCs>dVb6sCjx;TvFqHlWy2L=!&P%yWlL$dy)k)70bPYfrF`se z4i4uL-1o%80*hEbF-l!1P8`eKw6kZ_L>R!4$MU_BaXP@ZIlf zc~{au;s_rf9Bh2~KLmkq|K_rv?=MnyQ-IhaN^=KDJ@&Xr3({w^CY+~(^SCEkwbXL%pb@`K~P zJ;U~mP(rUTANoUsiq}b*qAwrvZcu$aKA%!;@3}f6Lzhx;M0+Z+5^aEKMrVG15U=6` z&pV0fVUn5gp9-8GDioIDTNUr~!(;PimmLfQtT`1Py$PEeA%g9;Yx-%-|f9JAWm1^nQHC>38;Vy-*(6;`y+CIdkweFPZUb z!V*pdxod$FHqqM<0rk<`>%f)5PqOAWsv~6V%m2~>lpHiW2Tf$RW|B&-=5ta|kT2n) z6&4A2*+G2{8Ma|nicS)$UCfkE5TmEJdx1`6!yiPp{)WcA=&rt=>zH&V)s`?A zhQu=ApjX#(Pgqp42MSoPVL5iO9iqPymW!(uE9Mu=$Ne+YZd=X`R((tJ$riaIP60jR zP%T#1zx;?rUVNT9;ztM#N7~gvYj&#(iwRlZsnhF2V8Rb+kO+353br z_{iOiV{jFrmF8G2A7@pm^gG5pSKr4r7I zmI13sJLDp*OXRz=x7#8Z5kFw98na97+_^Fr)A)Kc=MTldJL*Iq9XZ>6y^czs5DO4EU8dIUGi#d zO!x-|k#&DWjtU7U?9z|;*%eJikJ8dW-2Lp3kukJSPoY92jbb_PnV2{W#IVjl0EhOrJW9{-*n(nL4LF3e)* zX(XV_=4M?m zP7NY;&7(W?n{g}FJ=r5ap&eFyF_~rW_Pa;9_!hn`)!&d&GrKA6-wh9W9!2wh4zFMWEMId+P}|Awo`#bu6tiYxq_HbN)J;1v$9PJajF-aeM2i;#7o6Ze(Pyz! zd-g~8^Oa=@UmW<&OEms1aFKr3k-y9UW_1^xN!CySw_zd76gS!ib^N!IWsZbykUF;9 z>|6RnCnqQDFpCLGm0eC)$Qs%A#9%S2@@4`TOI#VLXie-V@_64<6XCVO3>zkC8J2mM z--y2t*(|-&-_M!D30Qa4E`qg4FD>%(^IHbgLo{APPjZGN8?2q)2y77rKqaUE1N_d} zD-4cRreRcHr~Etjr1*;W6=Yu2hQ%|ZDUH0sUurcm3+L==`F>6uqaYwG96tPhNm;|Y zbC|MnZ&{O>8nKFlXiR}!Exr=<5EFOBH_Y9-qVG)$FfsdvfTF$_APzXKWP8^^-r~m_ z@1jv|3ENy!9DnfiN9gnH%kQO;^V3%W?>AQT&2#168+&q`U zulubL<-jZ?%=^p4w{17|n8^!G)8{>M-vul5a9y9v=zB(M_Cq&R#rM*>-R70r@tj|& zR}#eJuL;QbeWpsRe&8|{3;JiEX-gT|J|czepZ!ZkL$8&sm4lJOy)L3*OHkj@-8w2+q=Uo6T+IP_c;!n!(Wkz_KnE>>tHnYm<#lNf zwjBSu#U!-#Bf~82z&$+|ISf{dCnyVo&j}FYBn`Hhk;Tca2fUpfkKfquKG1#_@HDuy z;dF)z8x;uzuHsSVgoU^L3?vx#dpgMVgyw88|2` z-l{r{pSER_P%VD5Q<&#$w@l*<%TD&~8xvludd;1wpLe|X6f{WO9ax*BzeJ{#Ct<%j z>jaUNdQ6933d?%N?dnAE6VR`C!0@{%iW|?@W-iB66?_J`KMU{K=ffOfJCq{!pMSzi{tiH-OiJ0(H^O3Q2Gvvexd z^0;ptW0zfI$IW8ax3@^o13*(fbMwKV3VidwWkO7T(c9yL?5Om?MjA6LBTR&g!#su${*{xjvl+=aCVtqO z*v6Siw_45%3;r#l%pR8zIUX1*Uul5!E3&Gb%&^R`zq5YLu@-NMcRMiU70zp|)?}R! zv%BHa7E{YX%{=ZT0;r7V^*jekp&(V!F_(KQtMM38gQRSPZL^g zHp__qY>+B{Nn;nsvkHYhema2DmrG;U7KKU8 zt&-l_)D-Uhbi$Q+bJ@DbSO9bjC^&veM@)cr&Ql7C3Zq$xtqd|bhgn6Xj{0qhM-K78 z4GSQ16bc`+seB0gE5Dk)|4g?APk2rSP>+UI!Nff&Zo7c*bdA22D%pXrK`p_|bBig_ z7gZA{xG(_fPcm)I{(%!gnI_#+)%3QpEEX=4s44y(Z@ZBr6XZ#iKB&odX$W)Z9LE0# zg+Y40(=-tZg`l-YI2=B>p{5={K@cDag1RmbT5AMB0HxG@{w4w<5r_TwlZbpRgIsA6 z$`OE8a2F<#zw|nyr@jM5tYP2c;QRi4y}mJ|KwG?eZym>!=twyGy8Ng68lVen=#W_q zHAP?=63tN?A355Drig_=YczzdUC(!CYB2xvZVkk1t0Tk zCG-eXiej1+js;(0y z^ne{ZTbdAh>N%*d|0VM7BD7uZgHW8l0%3&^J^2h|h{u5a(3c9q?2Te>c6H0hZ?+-v z!QKXF=-cYWl6M1KUn(Hy1sG@wF&-)83H0A zBHHf3xZ~{MRw@>-I6sF-EC#n!VnFVK>$ylT+<-I;baZz!Aa}WMucp&*T^EUlhVt0g zOp@%0QVPX<9+{ODI8F$fb>^N)DUoVwD*xdrAYd$s_QEtxyzs&cc>3w5v9hv)hK2?> zj#IbsXliOgB9W-;-f4zmpuN2v_Jh(3R|tr%4uqe04(hePLQ$?ko8=xQ#mS41P8gA6 zAAqp#?e`6ZLg?%3tLyzi07x|?VVTB`b3Y-lBRR3lCPXS?;rad)o*PJEuqg~F1U@v- zh>o~}zaCvdDq_`~aDBIBRZG&W6nxCCl`zyCLDZJGFqea`G^7C9;ttYz50fj!+fI@Q zA<)^`3EQ@L4oq8*p$q{Lpp_5T^`I0WqyfveU`PQ#!SggM%YfDjn|o#oDR()9Xav3o z&j)PBh7{QOej%%mbE);#$*6?bw=pU2c^q$d9S_U*d5E{wJ>G$^5U~Y}Z2> z)PaXm$%o;DU>Q7ly~}7qE4}GLML0plGWlR7ill@bJc3C68L0QZ4p$VQ4FT=DD2}}c8Hyo1cp4D$p6?U&+~ZRSfp9ns z$8qjzp9ot9K0MS2DS()5R<4UpDR8_siq1s&`pq%#&GzeVaPX5K=t9gfaB)6|i}Sg1 z*_Hz5y5bmYic}rit9J2|NG6jo41-A$+JZ?Es%y$CU}xfZZ~LCr%$h#D37bNGlYU zXEAc=3KrH%P(~EpM^9p?xA8tzGz$5pDO`E~8q#?e+D>5L_-XWa#P3-HN-1*#<0Drw zy_iFw1>zmUICZ=q(RvyxUuzVXr*P%kIF^=k2q2N{J&m)++ILLS_VBKwk9>Lx*RPFY zCGSF55p)cl#bA5v{_l0U8yFoQ#f^mwJPp{*NAcM49)zrid`N|$i0OB}f-ipUA`-`6 z#7AB{gT@etiMX?Js}N{WPWT5UEKXd-<&jwweFZ1giQ}hEqb16V0(zLsc!1TZE4X}Z z5^E&|X|>?QGmoMzY%x(nM0GEl&0=zL5`BGrh{a-c8;9B1Srm&!3=9m^?T}-h=V5GY z43S6#U0q!d&KEOH7>^u7B$I&}e*<3F2LuolvnY&w8!{4u)7}rmS?XexhdhL3t83ed~{W@Z?7&c(8(%&^di-nnwg2_!@VQG_oVXC+cmLJjK{7LfFMj5q;I}^a8j7hQ{Mf(v1N_2| zoyPX+*vl>B%4?s;um1C2$6ND7bbkDw;E#UoXE59reZXU?P+FVC*MIw;;M0HnEoAIr z{M0Z08vemgK8J+3Ki{XgGKN3@^#6k2dg&ta%}?Ox|F7S`&-~3(_o(92aIq77`}q6ce;o2Y*JECO30GeGWBks)`7OMk_R;;l z{~W*di~k+EQ{jg?uIz;`*-kv{`#-uBgZ&*0zJf~JaDlx@)~~oAN}w6lQ$QT=>9l< z`76JPztPjcBnc6*j#LVTLZR-5PW3zw#bU9(TRyZ#sZ@F}dliKoSm(|nnps75=_34C z`3UZ>Eg^sDHAp86tFZ(3q6Vp) z4;Hiq(`8hb+{_fd^6!5UKmT+86rcOX3ZD9je~5qa>0ib_`1!w&PyUbp6Mps|{crf> z4?P9UjbLIn10@|a4?l{tr~Bae9z4&5zss@Jup)@ZBQVV{lF2Ac(?MhZ8GPXEFk;e& z=X&r}^${AK)?qyJogcyRmW_%#s5zG7gT+Lu?<}5vd>A1^!F4_O_jNpJ6HP-O#N#Ks zV8Msyx(L+0jHE!k>k)j%N1sDa*nsbO57D-|mq!s3$$`i5!iS$hd(WvO^xcryDjZgpIzlV3Hi#YuwKZT$E$sfR@-O)HF57+acg#j&0Z0`-Mwcc`qD};a$K;XIX0#TX!r2Jw9-*e#wfN9&X3|Tph zD7Ah50w8cpmDifa-84oD80IeHI%*BybKwU8v@l@X_WIl}*fF=l#`^92zMDT=m)GA{ zfHYuP*3Hjb-xj86V9PC~syxpDObfPUR=L|$!7G)MU|VCid0TmKJs*%3v|+$}81BH( zs+`F213+$zJpirLw)qOmWy|#x4BM&FeN{vG9$YVgFaRM;7!O%HcC~p%pvv1;7&ffR zK3#qP-x}TkzULuO3c|2pSytt>TVq>}uaM=*2{)EsF7M)Ix%NHyez5a?lBf}QC3wDuQKPPdR`6UGz7nu(8|ERH8%D1mx3x6k zI3|>@AWZ|JYWdps*!p}I)#ky_zFVm~Z2RW1W#?-J^hVujIqjllEZDFNHDAjSG|e%rammbscZ2hXA4 zc^we*NEKYHAY6WD-C4L;n?i>?eK-;c)HVlZ8S7U-;9{;s5=A{ypaB=h52Q zivRW}ehT0J13$RyB*tr*4F2#x{67BRKm0y2nGAY*d+~Qa`P2BwcYm~QhwPz;gHj4_ zyzvHJdF2(%&CQ{;wG|)x*vD}6=+U|?yN=`3eS&ji9*PHM;`VCaF_nb<%yWo*t%zJ< z7AjmmFZC8Dk-zv$SfG7`V5Wf+EmSS(ieLo3;~jYuT&P|n360EDbf^UrCZ;Hf?3 z&>r-Cg6U;P@s`POxamScGOL(k%PUo%W^6_ZylV{*}jX)0tgIk-Us!X|1C|KP37;nKHX z$Lnuif*0>XcN>uR^_Ip5FTafO%kN`qVGV(_(A0Ab11%oj`tp}>X~D+X@A(9t985t3 z15q8 zd-8GgwM0>F8`LN*Pvhd7ui{(pjzXsfG2Awf;{VUyc}K@pU3>VuWxA%T%PO`d_uefV zOfjZ7^bR2;0rE)ll1B)S5E5z#Ngg2yq=&pmAR!O}p%bbxHa1{mT)>NGS$pM=@137Mk8z#ioW3p1L_KI%Y1_h4$X%H8gqxd)WBKYBucX z!d^BKPdb1yqyjiJlA%_b8XAZs6%4t6;?g40EgRUlbuVIJEmLOAV{BCk7FigawJ}JA zTiDyQi{|ELI@;P%9U~Y&b3PMmt8tiAx*9jp*cw974M4_IUdyOaBajju?Af)GP})TC z$Z^zE_>s|wwe4W@I~!^3PJpe5vD4;IS5u0$&p@4xx3hE8n{0Te0mC_hQeT#|`UwzU z5I&QVHHMlY*xW!va{yhEP!nO|=_)Knh=#3OX$vLMu~J+!0&8*)Z>@Tp4#mUdg~zdA z+DPoCBj@5Hh$50KBgt~F6Ydj6HrT+65B!=te{nxsy9NB)-sH7q8LCE)V{Fy_PpZT_ z8(8tqYTjvzQ8;ckM=hL5h0mT#G1fCN_HJFz#&?^DqzzmZW0*c?29-V=pM|eP6og({ zO41=;ZjE#|o8MT)+Rd%Vo^mQi_z*P*laHQ3K~EPATes2~&7g~RMvR-tNKclAEj#Fl zq%g2hK6w^nN~|Qi_pp8YZlWm_+2*9Is*=R^RlK=r4`R_c794jprQ*%lPq?I6r33Awdw3BpF%O5f4+Nb3=*K+OUJBmLORTY(12{nn>C+A|m`RB*q znKcbV*Fn{h5C_n+aaRMk-Fg%6zOxBU(*Wr1>Y}Wog40evbI^CLdE<3{dFLH$-?kN9 z*V(yaI~^SzjIJBQj2SZysGD_NXW7dya_1dCqrJTi!!X#nV>^j>f|{C9j2k~bzgT%3 z(Q@?4E3a_pop;jH(}T@sqokyS+S=M-=_DyFEk)P$VeQ;BnM_nwRUO%Tlpr9KmtdK- z0PT&JNhuKwlYpUXWIA>s+6u6bpNnX=9`Yv~4hMC0b;G`KyWj7}<#G*8Cy4-7S=v`_ z3xkxZAGD7|k|b(sYI^S&`FJ_y<3_;m>T1C`|a$ z-Q53^FH!9q|y5oV!rwi97$qV^c5Wy6eBjw8Av) z?>@5_Nx7$R<Wj=hHrkd>L&42!eU*5lt zamSxboju6Pd;i28;Zn}L=BM27wZ(X~9(KNTFF(BV@9YZOnRDR}x%=m5(Y*3LuD$I+ zn!*zEzV=godCO(gIu+VBKFi%V|A;5vky-l9uQMz2Hov;{|Jc+iGGfAPjydg8zV+W{ z?4w{R$ol90PPeIxwJTP#Gbl0j{Oh>$M^`eb%nsQIn_sw(pWpEy>Wp(Zr8dEXKmHAm z)?djD*Imq*B0GAdk;m@3mAn4(5?a-K7SHcw>nrcl6iFSirEBz5E6e}ym;CwZ?TkM5 z491$<`1f!A%1&bx7ysa=Ty@dWc*QK;@4n2hZu}X`x5e>P%;LneClcSVmep&wkhF~C zobUgFo4&aOpKOrmc!!66b|ZH^_!forr?8-|nXVo0&>R~|QCd2gSPLufy`KACdY2w^ zIkV^0Q|S|F+x{jSwgoAhdm-Qc{&g%GU&0_Nf^_>@{`rTyd9}&JypxVY?s$X0J+ca+ zW(i;a;mw>p%}4vYFYxDI{)RP8QDnzx&b1 z-7+3e>}J+U7g19w=k}9S=MJ8G;ExC$Yk6(uCZg7nTzJDTxc;k)@mmFqOo;VQ{h2#| z^(gZE3psH_7x&-tJDxc1Tm0~va~bWked-Q0QIZ%rW-L~l4R2xLp(sN(J$4sA{P|P3 z7JQKt$M4|DyZ_BD(-M9*Zz|?^CvV($Gq*pumKY=X>Mwr7e=T;i>Vez%>Hob(k5I25!{){_rI1{;Z2hZMpJ@>w}ouuku)?QQBWzn+>gBXEAqyS*R?lvh<@b2#w&{fDQ{ceaP-O;2*`fB%L}T}g_j z9LK^*RVa}r-d*!1ol-4lUG*cr_qD~8I7Grbmh;D--^J^}YEC}pFGIkjEOmy-^v%vuXE3RPqDYh&KK0t{NkMV_lDCm37THH zkL!QEjKaC6bMd7YP*G7qf#nmqt|OaFR8&<{UR|BfF!(rzT~%doV}qJpIYu13#T2B_NuD#?z@}Vv3+|klH4#1_U_ru`gLzH zdGeG4Mv-co#){?3>F(~zWpV4ePQ%XayuEI1-bs?jk$u}W46@lQ@pzncI*q1j!@BN% zJRV0;6#Rbwur-G2x=u72MV4he9?y|Gx=g8P001BWNkl#%=-w@}7rv%IHFgMp?Q zSp4Jo$`5bi!eho@%5<^uwdYv3EsX50VCqo|sBs!(vme%{LZ@}rL;UU!5AjaS!{RSp z&G&D)k^i`;9`}HaTg2`g$*kE^@!NYd19om)N4sSzr<`&OQ^rrAwz`niu9e*Li@SMo zyUzRzzRI~@yoh7QSXlq~J^c0Qjik(e#xI;hjnjmxWQoUhCY*2~-?{GV%q)@UXg^dcLYI#CRjuB|Khj2oL1?GZ4w)M=NW$goBLR` zJxcL}`JDdcuW(eYgLv~go_PEPnz~X57AIq-&Su;QCt5a5T2UyRbQa&c;Tz1a@sbHN zvF7Dv?270ZN|Mdb-ox+je~u3OB))vr_qhJ%>p1WDI)+xOa1*|o+02|&gGta(l1V6? z#W~;l5#PP~>rC>c*zxM$xbNYY2n;308_wFlMYq(GK%SJYBjIgtfte}!YFRAWkP=czycl1ElFQdxgK zr=NK-OBalyeeI(>^zd`EB~@BiKfqt^dxV{-LXN%gD*p4k*K_e{Gb#8440C3)mK2VtMr+&$6itN+vDlOW(MVI*$n@ zny%bhFk#v>$|@@l%f(P~lrZV21=PAF z6eUYMX=3(ySMrlz{e-WaIt`e)xZR;L#OyC@!X)bTMB#a}h0;aMmCdV_o^4z$yk=2v4qK!rZKL@gZaJWFhDYi=yDN9 zU-Dgk_@f_i-s0&L7JM?g%rFeHnG7WrOp zP>7C>j$z%GMO9TgIywl4!$9A#b^(1-*>y?rYV-%gAm z$ygjN3d9VtNQ`7p3PI|10%bx0!r5NuM{gOG_s~s}kcEL3kW{>j&FeQ4SHb2h#A`NT zvfJ@^T!?Ctj>ZmzLJ#{V!w3b%OrO7i8I#5_rltr<05uXK*po!fMA^CKIbMCM6~j}4 zUrLiuG^`E>Mszo;m#;^O?BUJjud=hP2dirY<0jW&kqfAvGM;jWl_U1T%ST7h$t|sUAOt64%h%QhG0rfVGgZQ)e=&vYg7X3z$~t2Q|u$mCI;|BnHV8 z5|AxU9L@qnHBKxXC8_Ahf(TlQSR_U!YalyHnSRWfOdsV$O9y!S`DfV`S5U)SX~>Lb z{`8SpB?xa`!SXflki>+`>meIUV6qn=s4*HhtzuV8E9;gmqoKP8tG}F))io4Y9TeBj zpwjm_GtA}q@jXAA&8qr{oWIBVM39)7M^Q_v`0p1e)Vh;&b$7;8u zMH*SXeB+R;?QQ63(&-G@Xe-bE@kVa`p9cx}7jV&q$5LYHl^@gl3`qilEb~6;5~N=K z6a*};LZ&V_fhiLvQeEam5)G1xI6a9T40?NzNOCXz&5j!8p1g>O6Gl@}?7@UiHW?%w z(Gf%w=RpQm;@vnP$A zWW;Eu&6`E3!yw+ik(ZWlC#wsXYy~(SHpE0X-H{ZE29hMAD;Z+p5GlRyok&Px?=lncIc0kdmdQe#Vxno!kjsC@|kXV3>{d^NONXj zubhG@vhQH8C1WJFzK+t>42GVk({Mc#2w>f>^sJ&8XnNk!k;h@O59LXa$8ckLkJp1q zmH?G(B0)TzK?e@z*eHl1CbJoVebH;Wt_>&w@R6Xj(a_LAI%8n6Sh3k1ea0=356T7> zWIH9JCNR3piAQ$gDFxJKmNhmKjAlqE+j#b#U()Cm3A9U$96y0_pNx_=-g5~wA&Fvd zgIclIz@TaB;q|5qGX9ara_t}Q=j6r+qTa@{4?fAu8`^pULv$UrKSV@0klEz`t7x+0 z^}CS?_cpjT3=CcG%QO+NIt#G8*GVNxm`uIKuH0o>RZ(;uE!9KE&TVvNbo7?DdE^gw zvD#tK*==U@)G1WjHH=7rJ$qY;s{%fglLD92e{(;gz0K}PXY=Dfjbd9%6hUw0se9jK z_0Axw-WyH4Z>{&yOAt)RCZVtKt^gS5nu?(rD9I*vG=$M~uoXD5nEK>E6dB5WaRpIA zmL)_%KsGz@`8)_Tlj&-tIg&xIupE%L$DVpEw^c8pGohfhy~w}TJm$u$wlZR!?MPp%Vd>j@`THj>O3H7`*-OKk=s>F0z3D zrK84SD)k`t?4qfui)_}wYO!In_V$Pogb&sipVQ-UUzD$aAPT)QXlQ8ap*8LsUJwu^ z8CjNkb-&QB4pj`@K<|mu*07!Kv<7?D@{iyCl2@!6?NK{*lP6PQQHErMjbIr?{o;8% z@%C;SS~l|7UDwmNSE^ouT@~aX5yMeM>g=HEI{EINgOf z?VyE|v@|r5U;)QmegnUmc@~{<4R!4U-22=sw1(1P*wA$i{Qx?;Wc+l-7Q1jc{kZy) ziS*>YXAC9Cn-Aa3hQB^eV#1d=>5D45!DrG#Cy&D{zk)z@O$|5P{3Gu9!*6M7YNEKL zg!3=Bh(*U9_kn*qYxZ2Ox%N8l|I59!wY5?)Vgy%w?dwdKIBD=5MwF;ucof(D;0FHj z-~)t0A;yiLz?Z*zIW;3k<`*Q7BN~zBT=MvffG?%lf?HEI-IuXos%n(EJ{8SXk% zlBA*KD0A2`&7X(kg&2uM6RJZ10n~Vio-MB}+MjKgM;-(CB2R)mhS@QR>SX>ULSuJ?v|fedW1bri#V%)P z88UsPWO_I5k9^w0;_?hMnCL3mOa@gGu((U9Kj&L~dA0`(0|O9c3-$s5CHP6qCElM_c5BK6TulC78nZ;{Pp`wTY%L*+~%m~7Zg zSsGXTlNYvV9CO~O%ozC?YnoyQO?m?`lr*V~esKIc)GiXDh^g-~W#}5p2b6D;t#)Lk zi#1RDi$-A_r!1b$h-X*Ol1TT4KKCEyoTFH9>~TE#(p$U}*vaar?!~ArW6V*NxNI_z zA(PFZsyc$z%b5Cex%|9oNc1{d1kr@W*u<)=NpFT{Z2?7Pqku165 z+ZkoQ5PBnk2b_NFcsw4gW`k6slcvUQtR62O zpAV1Qg+(;@RI1Pz@Bbm*zKI8J|2{vy>)$vQp34Pi)Kh4c4zS_%W#;Vn4#O~bZ(@sR z^)U6+E4lWXFEgV!$;!tc=3g(Y$36E;+%z8E!Xoue_2iWu@`dANz-v#5B;Jf zT8k+3m=Oeno_LH*R_UGp=ctyarfH-z89+dGl~Uw39XLzIP-%JXUVeT1jXd^tocU*5 z#=?msFdvBb1T&R$mat$-4OTtI)>j^7UBF6Zc?D(>K*sIyVzrpik}=x%?j~*X;P&|N zcwAU5B8VP*UK_Fq*>sXrDw*TKJo3n~+I~C3e)ji|F5?y}E}su82HEgVcD8C1xIB3G z(}zQ`cCVpFY3{bM^s3vq{kCg3X=XJhHBQsI=UK6?9lh_u^j_&TK2;qrQcyOAMW>v= z2!}y**K!_x^i2vT&85^bL@#iLhLTOAsvt@xyd`BgNU-s#d${d~Kjzsj5+`198S}>! z4=9m&;G>WDft+vkjNy#$e2)tjPr#gR=heUdnunL~Afx4VV0QAoov@`k~`i)PbEI`yCg}%VzCTsT`mX$vMgh_ z+lQ=kKoA71rv05H#z8wtpx0?;Hkq!GjXLa#(w)5H}_wwR~R^qWREnDB@#pmB3r~>FYo2p^xebK{uuB@l7>mSP?Wb_gzM-M~G z(!Kcw{`K@4f+-zA&k*ixp|vaC_d$hpGKr#r$?m{zG9ef`nPh@^JVx`5J%rNP-U#P@ zhl!yQ+`W%Pl5OJ-7NH|?YOzRB)ezy2~PMlUS{n~UnH zbEx&oD2Y8h|ChhHId4s=x%BulvU6T z9aUA1uzHvpyH`BPlgl>}$r=cHmT+4WoslfM0lJnYnMk1;`|3hHPj3}i5?W)Al(zBtv!Gtnkbt# zk1?JC!~wk2M=!0Lwf}yCHCx+I0X>_bb8jp0tb&1os-#J#m7F}u-ia(eaS6Die7 z)#$O57TNlYvJz$ErchT=gp>`lW#ywh|LQx06EOnKTUfK~1sW4&Or1Fqw?if!Xk%An zBk^pCSoagzCLlVzjGQ)$k#31htdVE#eSkGh-Nd6|8rMC~3oEvNCXL%a zzKlm$|M-KfYBe$M+;4Ns?cZTRZ6PK}>~+v1V0F2$h$4oTr6-;s9u2U!p_#bW%aFM( zyqALe?Jat4UpMI2r~7Tz`=1T^bnCu13=9Mi%wDD}T*ADGr5I`!RUXNl86`vRz@AOS zh$j_vH9@eenQ+{|>K@6=`4h3kck{|qPqA)i5KR}*GEtgayGUhJFm#kmid0{8iqTKA z!Bs_$)}rK~Z2WvKxbkZ(9P1*{`Zj;R`;RQ&86CzMc)ev*^R>G3$Yb~+nM~5u)I>6w z9M*K`?(U|grDaI=Q#)KxRh5>O7P`8+hTy?gL>O6xb=rJv5i>$xhQFbyq?@;qYTAUZ zWInP?G&VMpPN#?OslYG@hr={BHV)y&?eEPbF-R*1C*QiR)6&vHcX#)&)(v^QANg`y zc?>%Ul9ihJ^SJibES6v1!^(G7^V^$mA%4Xr95s0aRzV}$zJoQdy-AyCJm)VSi^pn^ z2?c0x50KFfG&N12Z4ZHNC#{`P6cw~|g3k6fqKRTUn|BjPsDMGDb1y9&AqvGPt*xCT z6$4Gm5NO#$ATfrCr(D3&7vJWAm$&oM!}n65ZzZ*T6}#eD1W`uoZe(*~J5`<{+FRR5 zq!rLqdID{<1>zL=o%HSzCPpq?%2{7nLCXU#)3ovdZv6TlW=$M{%T><83ofTv%+Rr? zg!|9qY8FaDA5g*P*2d?|9lD3*Tp z%a|R-)Gs}Y6P9n_pReuZ<@u*d>|=C%Co zj;nZQ#wY}<8HpsFTUYSfnuS!II1QIo2Ej`8!qb^w`z&c?Dzj%!!L?u0VcW<>oOb%j z?Cf}$ox7HC&#m8N{j_noT^`0QypZX|9<(JFaNLSbJn=>&%O1NIe`*7X?JH?cs=fLm zu#0VbyD0NkV;Q1NVhmj+*4au&Ac~4kHWi|6PY1DLH+z~pNhul_X@YI-ghD~OcefGK zbTG1nT6PnN&B0@}F)V|EL>X84L?(_crs16co1gzX56jz_F#BZYl(fqTQ7zRqTo84_eW{j?7!h$bw>e7{TJhq0-Pyd>0yVo;)OfeSs7*4%x9+ET! zV{8POp61p3^S2t(SI=Uk-HFRt$mlVPm_MToQ4q1$O=oJApVoJ}So6f+_)Y7}bidnx zq8lKj=-R!9P;3&i-bF`u6h#LulcckC4}mU)*6toO9kgVW_Vx~XluC?n2W?$3Gz@e# zNNZyUK|1LSL{Tvi(lI()+el6wjomB)BEE66Id0KxRy4iK%#%*0)-}|Q>jvSiuk!f6 zCXyI0^1{2G%V+?tG-SBqyj{f1ySo^&$?y2`ub56 z&MBd*xtUN(2SgH`d+7*7aoK&Cgx>GOgYC3-#d&<0v7=}SG7$lWS<({!vEcpHYPc*~BFpS>J=yV!Q(=ZHU*hWh$ib5un8P+^& z7zWvF7L&;|G$$ZSh?B=-i$u|OzDd@jV@Lvqk|DKg1ESS|wQ2%_{K50nb)9rNJ!G8& zhYPBz=IAgK^@1n}0~)sWoW$<3rwuHYx*=CJ(V&#`*l z7T$dH9@anarod{(;w)l({b~H?;(1hftcZGsU}p%qd=_V3HWpp7Qc5ekcR8?(s^_dr zC!vd0e8lN!Z=XCI=(s80uoe)JVI`hlS)+7pwb(;4Esqz$1(0AdL zJQFzoyEkDi8^f|Uw-L#R2##{<&$^fsXI3Dq0XkZB#-DO7WoZ@BSxPe0>&U2E@ zDzk~v)lbl(7c+b63`P~15lz!L|6AW7>3o#UZ5b4!h{YFOPF49tc1Mz!eRkrZAWD%1 zNe~dVZdN^bH*fr-e{&ZQWeX(}7xVR-ZsN4s7>t<9S(l7O6)Y4=Nm}>pCRsFx^S?X} z9TTNWoM<9TY58=%{QV!{sd|{@Z#NKA1XNo!r(AFW3&)lq3Ig6qOZnCf8vegO&Za#b zta+=I#V4H3SxPb8CJ&XPMk8nAz0ISKh}3qGnqw~Fo8MGEwC|^TZ#fZLB~_zFla55m8h+%V(!0e`&bX0Y+SffvV^bS7$6UfW zCtt)V3&&%VR05ry=>Ex^a9$;vV8(9>(-n6z`|K;Z$S9{NW@glkB^-Cuc!IM=^Ym*w zkjiEL~%j2cM{`&i*3vE3F_1 zxNA?~Kd<{11&^$xGb>^#n#_gYo=(lE*9mC?-hw#6XaYGFrl7W-b1t8XA=~lk?KC&4 zxF#*;{F3rSO*CUtRE#%VeHw3KC;NGMEqM+g3*BE%zN|2K#?yu`1rxt_nixP^?Z zoMSG%k=t*&kdbcd5!s_f6h#V4N_xwH*zCwd)tHyZ=N>Cotl+-;?jsh9z3<;nIN=1Y zxZ(7|$AI@o>eqe{PF@b=qpvwHPv+S=MEEiL85 z6HlbBt}frCIFF$VlZ-TfHjY#ZqjdvWj{%~9o=KA2wg%B+$5LJk`O-cxJp~lat3rFKF7@l=F4Z~pND=*R3+Kk)lW%f}= zQCL#?o`-ZYJrRQ4A$l?zlC=O|Q86CJr+GOuP?Avsff$n0hqu5)D&B*sz=O?v@L`X- zmL}TWO)MkeE-c1nHy;+G)~2;<*|K3h7ORcv^X5@gSNG9ps41i~Dze3bNfH5_Y$}C< z37d7l@fkX)Sb$(m#py4?WjDS5GOs0Rc=bMRxaJPtP1x{y97q^wN{Uc>GaV6`Sr^~T z@Ba8LMjfOpSXYvSyMrWD2~XhxzHjJcdm;ovJ(yg6Ja$OL6>M%7)`Rc}G&eS~>gAV^ zB#H4;r!isr^iQlls;((yGAbsk6nWKwAavkkLJ!cbB~f&mPBAs(y3rYC>HM(4SF@x;0V zMk*M(N;29_B%xq&72ZLk5f`s&YYu<#&iTkB^p|mZGXFpbH4q*w8XQ3 zh-c)<{Lj<(bM81FM`ALGVHl{2g0AZbnt@4_^L4`aGXSmI+Q~DoH@oiS_rK?y6QU?`$t9O?;)y4IU{t@RX*~JllRWm=W88JuU3k48 zy*YI@o8@Oe`x&?2emm)O8jHok?|=V$zWBv24&6+GkAd#)ZZeq+<>lqWwikt}s&sU8 zV6j*zDk>V|R_B??K6~WU^u#KwcTEqoD5umZkNr`}z4D55Vlp_}Uq@?CkP00Hkm%CN_toZ+|Vt;TwFfau-$bX}boMjyJjc`?s??Qo)zM|6ka5WSKl| zG+w*>iO9B+&63S#QB@UVP;sBtm{BovEo0fal=kw)_Yo8)K9uJ0L z4C`jyk|a@7RP_1CByu}3)z{;AsfR?O1+72=!$6G%NN!()Xtp7H4(tfBTCG%6R1E79 zOM)QabULxwY(q8>;lP|_Ns>n%c?>z|no2sIB;6AymPn#e zfJG9}bcIADjd$F97M!)1^1O!2V^A0<=_p-+D2irakRcup5KU&W+RcYAQ(IH$Y2V3~ z&FhJ$pQoBe3a6b-t=E!Yf;{rbW1p%BwX$=|Cf4oULE-%+Oqn{3Q)|jG50!hHo+Z}R zM!2W9hgc>aCJ>5{F0md|mSrA?2ZAJ1zTj*wJ!>-$E#FSYQp#~(y@Ul71&Dd_eW}Dyf`T1OPYb(SLik3PU_M+%J=sF}?6bM}q6lxbWTAKlN7s+RCgl%e zdE}ADKA1<=bKzxQMP9ZU+2&$;{d}qlZ9``KdMe1C9RUhw9?!`WRZN8vdm9?4ttq6! zX(F%jKK-+jqkI-$``JAlzqNs+y__+%Rk&^Wegt{s@p0gGzdz3~lgY4Z)hg!9nS;~m z1Rx%d^V(~#v3BiRR;*aT)TvWB@4WLk`|Pu+tgOrv@-%&AR9sEdZ4!b62<{eSaCdii zcXtTx?(Xg`!QI^@c(4G$9fA(-+>_`1?k_^tnqlU2cXid?yJ{V=^u#4~S%2qx-Ylghpv_|)0Kb=*au=?I`mE`QO!PUaux0(|?9AHP z*)co;1#C1JudlDGsHyz}>L8dUna2CMC4~x*D@u1ZJM0PIcnR3y;pY8O1MXRrg*~Ls_aqG&8#6y z%CDXCtR~;j4uR##+i5YY-Fld)HJ)Di^0^bPbe*Rd4=!+Vw0M`kz;<@{*%x(+!`g8} z@M^^YUCj*}^)O{$^55$SUC;iAIyo_KK8&zChO$*Ij$@l=_ph}(t#o-X%kw>gS~om_ z;`ZX8jv2ESEb1FG zC$~n;TEOO{hzu;G08Cmw?!N<102sN0hJj(klmYYkQ~O+sYbSo|sw@uyd4j~vcSOaZ zi+f`%sBL?Wduu_#kG8E@0)zr&Nyog3_bHA_Xdwm+%a!lXQcoOs2!Jifxn|UG;L-_c ziquyuW4|vAfEQ9BMZ0-aWF!ER-rU^mZJgXcCx9XVJ6ave`8Q7BGpsc>(Cl760>G;~ zj~rkPR!Fhq+F8GP1~Eb^i9$+B3V1&aY}=n}11{28Q(}NCZAAq_I$(kV!id^G+Q0P# z-c5lb+3UA$>n_+mT!a{uG`8jy6&2;Qg8#~LRB8ht7&&qf222^3kKPoKLL7M0yeDAy zIt>~T;)FH4Ni*hnN(TYw|X<7phHUz4w zef}7P6LWAtD2W2R=mIxwVt-)Yadu@TLwnS>XUc@>qNsZDJ1G|YL2k2FwR`u*$1VyO zq3L>5cqk_;+r|}&Y@S$aFX*}H)?eeF;U)WisVH@B5>*S2t3wnf?+#nANKvH}0&^cq z=N}XH%KZ?Ahlc_B>7&&-*d2jF9&o-EA!hV%;mfmA@c835C8}85*5N2afkm_mF5n+d7BSn2R%j zT8xR?2-zFldmkir42%l&-Ct%x52a4ga!hbMwefWOA0!1Gm)uz;v0NNZhOy$`;5WoY5yF%Qc_Z)33$2g zI~Tw29~H$-r?hYa30V7XEVuhnL0DOB?P0dlG`7N1dGF1{^$dUBd{%&fzhaTtK768D zC3@VbFj*4dzPJZp4WvblD>5uNq&ad$6@Nbn=Ne)G3xE_MLP0?Rka|Y-8q@Y=CtL_& zL4Y2d0_`s=$`}){r*)SiM2Ij_~vA?$ewpgq6F>D6M2DmwbnM-nH33>#OVu(|-6 zT&^?;z~?yd#Fkt>3-)c>6T%4DA_&l(07S+CJUDT|>XI!(hSRu_5C&-MoIfruEhT4% z0uJN+u?W&<6;)C^g!?<7s}$Jx5O`E(jTwc7g==k3Ug_ym2WySbaTEKF338>-zb0&3 zjhQkW_q(4gPaNAFZ!Y|5XAXszT!1hUYPfe}#&7c+GYow3()kQ0hMV~bqk6)LD}T=A zGgogu{ps0xyCGMu9N_L_krkeA(z)px-6aC>!vo;>*>m897jhT?+jhvdHAw>G2zBws zI%C!d;jWl!3JVJh40hloXj-x6>^XZ+1xdDZ0$V?usAqF*jz7%Tvt5S{BuNY$zL9;$ zcF!E4=Ed;-4|jb8$vYc8+M$Vp>#qo77TM9QOrudij;y=nga8Sw4OfnD9?xy9@q6Z~ zbqk)~DSy~o4*?Jg&2{+Wc`{_kvgOQcET7(*T)1-n9ki>x`{yyU#-MTK_sk1=`PVK` zq>)_i#zd=cZ=MR3D6qlDT2{}dIZ2)#t^7Rxx>?^&Bo$HsaU^@UP+#mW?fR|3_O(N5 z$wfpF2y8$W4)yaV^jMN17DVp4pU#*Fo+i>yoKGEwNb;s%!j#)N@g&qWtJQ!s!&axF z0>~dEO6P&)xSwtL_hzNmY+0@3zS9Z7)x>C>Z_?=8+?ph5el^4)GDI@FF{M8bSi( zC&N0V;?Tb9t~1bF+T}*Djuqs7FJp$P=SOrPCN5xMUSKaJDKbJ!k?RxJ?lS7|ux^CtBlRP-Oc_?AqbY@I4# zewaVCEh;&%duv2bf#rQ zb=%<5p9$ymFOIYC#Sf1i-r1iJ2Zr?u8amM&5+oXf3`gfjSjnx(pLJ|8=~~Sd$PLJn z$bQXo9$|qrN7OaKhrjb)Vmaq%ut@RX;5+B4b>io3@*nY_i4x(#$Mg0*GG<8Ro1`HA zFs&#Jli&#ElVEm~Or@lueC#uu-WJGTif4hLJw<@d?FrxRcQ!NErq1MQ>$!e-<>2#g zRUQKs2pIfgOrIos5giV?YN@dl88q}n2|B?M7yL+&#-10DJQ^>+lworAr+Hy%>08~V zgx}|fE|2TBgG{T31NPpr3e`%E?O@bP&+g|t3`LW-mq#F1CTaKC$}rE}PEpr0JYBAD zH#gP%*MLIt4BWJZ6il8s7C@n)hz8`X_nh^R0asU!fE{q1Ry9nZ0FWcd)kYSEQ$!}G zrYgO>pLV;s>nyn(J#*W53fqYgX?(aNi!7FiU?J6>N;KuJX}~MxaE#J-OyXeQCw;t1 zJls1V1G`w0yXsuTNEd6gP?1(BCY#ht8r2$n4BrSiW!B!=yC@adnu`=0MHhA zhANWjoG{P*9dl!F$OqNu6KNhvM9z@3N=ib8hTEdMnG~z_Gis`;ii(O33D%gjS}c<4BPQG} z2sqCft7ioC`)rDOm>WTWt{-?VDk{#ifB}$xY()hSg$pT;I^;Y&IDsSv(}!)IIgi;K z0l023d7UB?PuMla=7mcXHrvAWO3BVbn)gd#vgCP0eotaB+7fD_6ONXI?y1F><&Oq*}syce8Oo7%>Oec3ciAkejpU_p~%Ey^A z9er)N!`z$wSWGeg;0@Fk#5r=iuAQU&tINwAl9(>KGi9wp;n6eRJ3PG$-s3FdbZ&fK zPXyQ2@rzN&g!~833=M@z|9|6CWO)gF;Vfsx#MY| zivuY^sGTau+k88BN)O@g7JOh;i-OB*CGT&BTwWYC8D4 zT4iy;Y=N5SGk$e7-k(-><1J~JfYHOjeOGh*pbR;umo33RSu>{26_piIXMvDa?WV?R z>#?DOr(S2x2?EkkIs}N+c6#En7~N<>zZ`l(S~uNEDruytsM;*&s~;LU5`lOyig;Su zP5SI*2pA@_(CO!@wn9HYxnJKdg`3)o!HLPudx-C6oO!UYvpW3!iA*yw$C!CyFom=o+qR67JMHjOx@jYZA!`+9i{OGn#`Q{=zcyrIUncpTO&XOZUa(j|H8`doFK1zlV;Wp0e@My zuC%Ss*4m+1bS~EEl7FKAY1+H)2YBp`8R`H+=i8{AKITTGPQ7i=L%HMW<%znON zIdwK!xv>a$r)!*)LDxdsfSROrmz@dPMY$Y}JPxm*{aD3Sd%i7v(Q0WdtsX?r;rqfN zEG)d{JTI#Z#fdIzxYA&1z0s8s5D)-P+Do$DvG@X*1+c0;{bY!iTq1CC)!SRfb{U8@ z)zNXc8DtoFmz%NwAUfX^eUU7ax7dhhFa`>e$?IO=l`+@J=%JoBr=1vRv2oVTGg)Qw zSxP72S~}l${P+Jhs|5Up{9tbSLtpwsf*Lh`I&n-^*`OVv(ev2qEv%^~2ZTYO#5K3&A4b7N4l-87H^G5{2lhK)4~McMIDEsTNb&F? zWtfkV-?y$j9k^N0R0S2uN|@#CQDFiyuu5*NQF9c@uTyD%43P#=C^I@ZP%?k5g(5Qb zm=d8>P=!?eaCLgDT2x_t+q7;W6e1IIN#oUoCz2{|6qT^}S4H;b6d5;L_1-2R{V$HC z*JQMLjxmXGU7uRRoA3BD5^7C?WUGwV__Km)h=L$dOjH`mB_z{klqJb&`6yvkxoNq^ z-9iJJ(cVR)?qGw$X}L5y?5bEXtwN*DV74dxxQM3NcqZy6Q`l}CYWjMU6)D)r$cAwg zZIkX$e*hb^i8I#KqR;s20fRf-A+jQ_9sQGNf;AcZoW6teTcv~d>6!8%tiqB;t0B8E zF)`tyQ=xbG2&*BvPGyA7mof+IYlj(L7}CX>_=t=YBk=^~MnYN>>wNr=x--h!0v)MV z^puXL$Azh^IKGgn>?>V6IEKcP&n=K_E29Pebz*tYnoN}Qzmx9}7zI-^0>fK)nfTj+ z&*b%k4h`WR`O@~@2o_sbdj0r;D4ryzZ9cedub`q540P|Gxk`WRk-mvF+osiNV>sy# z%!oDB1X`7Vk`%atnXkzvq8!#*uxFfRkQSlpyGUlZ)|Uitmg&4F*ikXL%x7|c)Ymgw z85kYdWr<7qy12SF7!5~u@d{K`!p45_5uw$NZ}?1S{B+j|%zWwHlvh`1&r1pb)GMm@#`ykeQW^5Lq#H8|Jm<68;&R#B|`L|<|kpJ3w z@aH}3jkeo3P&ysb<`*~d>^7tFmU@`3Uh_OgElEtT3}hl!n~6ry%qPXDFbM)0DF?N7 z9js8AR+k^?W||G~GqztXg@TS!Bvo?N z(&h6f8qF3at79VBC3{ZYU0_mkkFbELD7tMLb>;5dE*Q9FnF^DzN46L6X)H8E3dydI zZB>;i>F9ElQVR?|PN>z4dH&frcHDOGYS0sIHk5Or_4T_;)GG-5p=$)KKw@bXx7W=Qp52YiMv}k)7c{P^DOysxB>S<1lWg6;*`! zEKJdY_2rXE-MXeH>@iJPY1txb*a1anYs#Qkv~_qwXEV&%e-FPgt`N$geTklvqp2 zI(aLrh_PT|xbEo*tPBN_rBkaXE7TFfAkqd?T;L=9|8%q$#ppAyb_y^Ai`_jlsN@Vw ztDCgCWr_(3&i;J)X>suxvfxu{tyU82&uwQe2b@^Sk$gp6WpM_q%z@u%fzg0Pi2vK0 zF9IG%UxVocSxUr5Uby_XSD!=M&+*XB0 zT%=}iv;;?WEvirXeDm;Uy%wKKp{}fGWM9*%i|2Ie&R)G`>vjBGR-r!>D1{Bm_#VJo zFIVI&qK}VyVl1Vphaa=sjg<_Rff-xW+B~HkCa6tozr8NCuJ$T--rGrJf{l_YSw5jq z))St0sjxbfYoAqIEI&+LIxO;i8=xF5xBQ!Yuc4%s+8n)@j#r8k1Oy1Di08IspUX#x zNo|JaK~pMCstPGh57Hl|=;|*#*>srAPKUZX9%agG`G_H~FrNd{+zI|9_g0o=3qO*Y zaHjC&$YBDHr7N$v9J z4B{f$oq^P>iQ&-=vq0MN2Acl+Dc6QjUFPCUl5{YM=|X=X#}-fiE%sqd+X10QbNv0< zyJ4=U8Df|`K@q);u5+J^WY{szf`;EFwB{xQbggjr&4}v!W!N|+0v1I{VCE0kVi8B) z_zqWEQ%ciII_-lh*;smWL(gWXt?wrTdQprN72}H)`gCif&1Lo6%ICsaL><)>?e^d1 zU%ZGElqiD-9TW-ADn<-eDM}&^>?PK7i8^Q#_CBSW&aSV^0$KO<;Y@E|cFfIZOIzCm zvH)wC|E~qG(tpN@!k{f|Xh{4ao#uGI?l!aLD9$OxfR%l#c~DWIq@lsF(d8>-H8qC&SP{wyHmR>;oTn@t zga}r~0pcZ3O~g`~1Yl4NU>qhxEwh<&1>d-(o(^1=_tBNj@MLl4MfiQus(*ysbbMVF z6;=8TAYp&3Wzj=MM#O;8H546vEf$$0)vX&+rlzF4yu1Jv3RIL=8Wl zK3YCbX7B`OVBWLtxu#}>!>l2j;KdkkSKX_m-k)`; z#RK`v73d2P8t-&euY2~Pz8`Mm_TbC$A-{l20j?qGG0z>;s=jy3Jbw30HvdGv6VbPq z>acdZb)K~jmX80Pngj^Do~DVp)PeUJ?YkH5cYVY$uk%4V>fR24hzv1)N36rfnf$x^ z0(is(&x@R8<>d852~gs3VFeW{>&i14cBr7SF$pm7R!M_BJvJ6$2h0JC3~FSH;?I$D zdVBHEp~D1nk;dJUAikR^TreNL;0=2c9h1OEYBrhEAp(_BXmQEgX3pdUaTubbv$G$y zwQ<1QQy#muw^-(x21rY*r~MY1i5-Aj0Qy+Q$Hz@lEBA8lJAu>Z*R(iT7^A&XR~EPd zvFdIs&uwgzy7Nkl$FnB&+M!)`6B{ORfqOGV`UF zGz=r__2^M5F}SQo#eQFS)?nBIlbiY0iMV6MZZuV7qgYboJjMHYq8hP|yu9zQ*7g)LSUyY=I$KpO?CM zu&r9IOI9W*EQdvS9rptE-%9*`3&`WH@VfqH`nmm)oW9;^iA|6D=3_;wHlDW+e+_zX zf8js`WfP9mZ(o1@hsu}-s_g8Q#u%SY{XG+NtVZG(PSY)&kE9bb;`L9foQ_9WL9W<2 zhciegY9FG;UGA?Hit-EO*!KVL8)XFB6yGYOibq%$ ze&Y^wT2;2l!N;uKQj(1J@#@1U`q63<_is~V90SA?%|zxRlL8uK+C<3UcDfIHZ8^nP zCx@?_f!$Wn4pRfJF*n(=F4<%U&bzuz9~W7@yxBtyw_H1~d&$QjGeDFlS4_shK?*gL zAfe2t(pLJM=sl?2oQv3H$4T#Vh(zE0fMbQv)~N;CepVD^uWNAVV)v2HihtGj5-TB4 zjEzQLQWosmQQOZk2gZVc2r9dUTY1RsEQ6PknN<3PZ~IvHzW^#hmfzgG<9&|ZlzMP# zN=E~?`=F?hoIpUjpLV3kQ~)hm@?q@ZFHOpTyVth?!+6h4U6?$5K7c?V&gf50P9_Xa zZ+?|9D0X-Tx}?o&0rywaTcx!@T|8h}Yyi4$R3(UU@!xRSHfB~sh);jjGmh;G-_ z6eI3QID2VmTX7@vmM<~e@ofHbw)vXffcOdwvE1^MI%{1yaLrFo^Y&pV_^xg?SYgS5 z2a8`C%WP6ka|5j?{!Wo{RNBfPUD}|)$MEm=nDhil$P!DlazAeu)&y4B(PL*RqQfsj z^GFDXKN^K)oIph)s*)Qn@^{gR5xf#5F!3@2$DdS_5QPD2Ix~)P6qd9U<~*kl~jLZ|y!|dkcVJf!-DHk|#GO*IViLZu5Aii?*v@z~pJj zj(T+~vaXjSPV)5^RnzLWdNP_nqvouP|F8|TAPr0s=DkjSSyWzQb|7u5+jr1 z$r_wsuzf1ZPwr@M2EAWX`o0WRtMJ|ERS3(8ROAWM^N-Bla1nIq15xDP{>4X|;lHN| zL%{EAsfrRNR$pOnC~!c&-*Ee8+bRyrjaWxb1w0i*7~Jk-G#OPW(}Yn@}_guIs73`+|+G>a{!kPP*f;VN5ob@k^@A9)Dk;1eZA5(H5s^>*M>8{cZ4HLn)l<-af5 zROM(#`xfXbqp9eU@;XPW=dKI-RE6dau5!+@325LMj4E%EC@Kr+<_Gl`2qqp#KFPla zZYPU19MXimfe!EaCnjdF;M0mZUw><7AK?(wZ5Z57e=B4|YcP`h2jek6uf-s>))-dG)|{tuU*% ziRF2c_|u;N8&bRY9$>UPDaWVlUwA|cCRBAGejq<15k~;u`n`!4&Qi#gMj`TC3jPLg zuaj~)qR&Dw{|Xx^WceX%0-TcW?wCgNY5R_2wCziVRMUFL#qS678hzucl|SmPg^B9E zpGa)H`nn%b-jpO=u-MLYB~1Y%%TahG=;kO(lNLH|i#MF0iLsJr1pl3jkH#loz9=CG z{J9WZ^~8-$b2GYkHO7d0Fs%AXIBVRgfNvXJIDUL9G;z(zxb81hvxE0FF`8dn+F&K$ zYK~2!1`ivQut%O5);`gC+2a+PZ>=kC`qKq$j<;Xcx%<&}-6eFL|B>r8%x;HQ`@3d0 zg~Ul#7)J;9l&t5++HUt$5?*APSny8KW`U;_r72&(tks13egd!e`?t7DAseEsw1;&4 zs`)>wLm9y2r=Qw_qCckW{7mThy?5~Jc;19J962efDGvq_X^iUK1MH>M5CKdpALnCs z0aO>dHp7bx#rP!kt3fHr(DNPYi6=hupU}D;7{lut?#qv&vkEMjnUz>>V_H0>xU+bV z7Zi-BK0}?Ggd2^R)U#1Gn=qwsWqQ;&k>~xFDm#>$!Ic*16}kMc9#zi4A{W|29ljyh zR^JwvaOY3K_&&7#x}MCrb6zKvk{}UW-MZPo_T-Va z*0tOcW13N{WVut11!z(3?$}!RTVqE3z)qQ&a^R6#J@ey+{i6<@KL?PJfvU>fC=?!d zmN&}eB?!qn%SFX)7NDSi>p9{wwolw_NK#JQI`QGZ2uZA>n!*b$w$HFCKS3QCXojVr z8(VM6ym2;zj2&wL=`3J?;EKxI0=IcJb=1Fu%};)%`wg#yiBcl5$y!D`-)g>-UUuFD zDq@d#PfjVZww7ZpMf!31;++$CagR)96-SxgJLB6Ox0U5No|wAV<*+^(e%0ac``-#0 ze}DWuEG)5LlwM*p`oaK=719=0H(dOk;@#X6` z3hP&1h`OciRrg4l(KoxecyCAPA2=(cn$Kh({J_jRH0OV1)PK6VFsBV*Y;nM@EI4y) z2lNujDS)04K&)#u2hRXJyY~H>A3SnW>+lZEC$X5#skHpqC_P$HiA&o}`b%EF@aDIB z915hBE$>JTdr6+4;{=&f)+PzWzp&0Ufn}w{n*~7%@J)bj$?G{GddHcoyrN>@*tLiz zH_X@o;(WapiFt0>_G_?g$30Jdk{6h6CHx>GgY?gf?>~Ie!|xk{tY{WPt@42cFpLIJ zu2@=9B7_M;Nr95$fq+Y3SHruoFe5Hw-QZgQp2tI*(hMK<3N9#FVwlh25F$ubP1*UP zvSS~nP-3yB$@cAcxLX6Q6=uRLJMasREdg^|8bL>$j8cT)&Y%s**rzyjhQawxQ79A6 zyeuGN>qGMtRzk~RglkuH4nol}-P#+h&YPq2;PrWF9c+LpnZ$@^#MfrSl0p$~JwX`3 zx;GLJ51~yd1SDJEZ1)+x;2a^hC-CPAdk<-aTijnm8ZeV8$s)x`?rjj%tTKBqY(STu z`mG+?RKoldj*VBY#v%3c@z3xcC@P1?aoegMF|UZPaeu9z5s(76*7Sv4CO1096M1}0 zE2&P{*9J=YuW?^?-ygUKwi({RMc*16_o>c-OOfU5>QiTA;nwxzN|z~hNQmgmF8>Sf zXxujd-PmcUC2bF&fK^*xn~hE^Emf zE7%>cbv0(@FsnIM0Qd)xST1rERQ#<%MEeNPdW1+=n&cH!<4@>qMT)1wy~KK^BzRN< zOp+}*LM|R)mbK`=_EQ|sfGpBnf^2NhQJYd)M=c)EQcb{9j}5yD0Tt^-)`@gR6Y}99 zTzdUAu)g_i0R!WzZQiC7>~$l(Hd!Fz^_%81QYQOU=i(5HQ3PlCVR(vi5{Zx*l(Xdk zU{$!g(A28PG&*1gJ&d1K(vPtc-k5Ki&q=W_$Y&d&&FzVDuVOq@&t+hufE7XUc}8}{ zDJ@Ok`I`kspA?F|LXpY#vl&)D}e`@tag?}~zlZF-*sx zxZ!?T4_6nvj{cxe>9N(SG!>sfmpS{i0z~fIEptU_tVt>d6%PU5T?90z^VWyB*B3=D zZf<~t?vZ=#f@Go7avsKgH=R(w1k3aNEKefWXtm|-aVw%MM_Tsx48f%)vbHYBFTcq3 zL2lvQL2C>2LdUeGslC;j22Psg0{(}|JGPsN zK=r2pA|?R9De37UXkGmp8ryJ8#wl(9tbL~xCAR2MHlQzN8@B73LvF(DM5e-S76jX8) zR+M0=Qj22D+1EJQPeQWBOot=fdILJ$zea9P3B`49OzIVJe?E+?V=oMmv~tTTW)#?ikxcs|-&=F~ z$O>pmC8Fub(jR0q9_rY@I=kITfYM7;2R5UJ2W0Opci!$oc@iE^U7nAZ>vgx^JNw%U z{d?vXI#+|Ls_8^(HT+k^?c2wi>+?A>^tfM3u92xG*wi>w4%lgyOQ)Z!!|v;P2m3p; zmlSpwYbur3*pfuTo-H_60-fGq57667Gw{eIg1wF`F334Mj#XHoAxY8EFx}fyf3L}n z9<`8UfAy63kv}V=e8`rb$}O#NGh}r0Lw4kDq+&vT_@KH>yVlNsNCtj>-8FXJT#J#{ zltyPkz#|~eGIdaF8DmT%xZut9kJc)Tw%$Rlj%fe(SN*bEt^705s zI1C-|vkMs@l?3!hjauZ^t7HIB@PvAuY6__0>d4HhJlD&#*|t2W0@pnQO`74*Ai2@W zUH{So(2BXsZt|k|RSX~v0-i5-4U9};kxemj!`?=!$o`mfBnQEMaYY5?`<^eRV@8S!!Yd~=Ls^PO!K;mG?SYtz&= z!Aq+jIswPzLjFCH|82DNST=@5)h(@dtM&b_mG|59;}VU8QE4dW%^Kq4Thb_{ZbNcl z{FHGo*YK-q^P1}3ut&tJ7tP!h|JlZrJU0nQd_!Qt`U&m5=>;PdDrkxJgDH8(%I&h- zeinE=Q*>*ld)KFGOkTQu>wMVA(qwY{PPGFt%yprRNMA&Rpq{bM;PLjQrzj%Aqe7s0HW7PsfWW~-^O5u00>9OT~N+EWh;vI&e(3HjZ zVSX}0+4q&nStN3v+jpqd_Yx(_@7eF_GAxbQ6XELY{SDgtb=5ZVeGiyI?IrG%K|e zppz$9s9TRs5_>`LBlVj6=>wfg7;aIWx7g4AeMe-N&sP8@6p{f- zQ?!K>!&&ZJw{1U9lxaZ)FrkcSq)&!f{sM}uDyqmy5xoH$SAy@ROL}H%q8wU6;;ru&iXY~q7T3a=Ml(;K@5VkilYq#B_zh$2Boqn zLC8vjjMft!vEUDaU9^(KM#dcj7TJB5Vu~zZ*AHgQc{Z?%@g8GRO`-S)WgpN&>7 za(i4wC2wMnDn~4pcH=f>}=cHnBe z7dXG`#op4UCK~e5D$Y+d8gA@@tOL|DJd&gPnK;a1^XF%$oK{N8`os25T!3yNhZYH+9?RF6&dPeXF@?8ylSUx=Vj;ups?TN)lDqGW z#yM%xY`5C#M$0eN!yP)Dq(Z*j#!4>)i%c<7)BUQ8P6jq|Ko%Oxn)E5^FJoaA zxT`HE+|bC&1?f{`Z_+Y&-92qR@|=Fs3LCg38L)uHkSYI>vvJ6nRT~*E&mKNiRiJG_ zWS#}6tYw6N*w6^y|AS{>pdsQGVfNJ(wHtaS9xqbQT>MG`LY+AU%frDkOfja>pJ}Jg ztsOj-B|rNV^KeKj;=*wyHb=tF!rQJLfi_6H?tIEO{c9!mgdqJ>-Q?$%6et;}XiUYX zB>?ploo5h&3{I^swm;+2`LWM=!KH?meetT;n5u3(?#-^vULM(>)aSl&@=zNJUH6RY%( zyB=sZho60DK}|;JA?)kXmBlhp(jbp|Nh=GBt!w=kCcmo`{`}6)sKY!jV0yjn4!iG3 zV+MO&dxQILm*@%J3~HLEEZ2tDlt!AK>mV$jQBKFVb?tN9&;YJ8GH~Eg`;z+&CjE8l zto%>TzTc~M#*#je1g~0uEVByE8@s@N^^(Uo3#gE=Xd%oP%x!r?Y@^l>wJpox8W;xe zVONXGpYh_dt$xtdBuXsS{b&d+)PQG2>Z-X&-Zt(vI-Etwj^Z0yD@;(VL?9=HOH;zz z7&-BZyDZSgEx)3|9AjH?2^a5qzv$Y~9lUrSCf86p7=P~ALWuyihL7&TfBibUxhZdG zNF*vM+Q_DXf)EC@S2T8THrlM-+~4oob5PLGWH(;}I{MkgMNtnAj{}#Iib5K@Ax^yh zjaS%NF5GPu!UE1bcSE$e8IwAr=Ui?tDkA1B>oVy#1{5NG z{>vRVWxEY^=w_b0-A;m^@=sp$*!hlY!b3MatT9`LYTSHGVZiS!2VX=k&uFVpd$Rpq z_%jB*=wb}sl2yVj_K;U7$MvMGCqgz6zAPclvo3)#PSTqszoc^$DxXVs{~&aE$HhmU zMOOBn`p&kd9Eq=^9KE|As%r^41fRZ}{$64(#p1cdP`&pH7uZEn(e2qGZI9Wuw;z4p z7!7zuyt>$YQ-6AXe(1k{%ORCct3Up|YPJwc%IchH6W1NxeDm>K$Mo0ByxaXVy>ue+ z*h&zzWgO`-(AVO2$2%tP+m#!~|I*3hyh&CeG;Ni1OE?FApR=`>0aFk{>aJ6S05?Cx zn_L1ib)cIY%_yD{dfl1B8gtGj>3)S@rPR`j1%G!!otSRAf|NdNk@rDUH{ExEO-^sh z<`5L;kQ21z6g9?m79HKM%>T(PGGFc_GC+B$w-sE!o4s~Rv#Q4_20o}|h;zL=<&Y7& zR#xovfBI{*)&cJ*4N9!kAl1T{;kky7<9nijfcQ6Zk?;L+rqOb~L{rlfI3M8}G-8&P z$5p>0dV2V}^e(Cp5}Ypo!VFzamU9{j=w!>ns>Bb7H<7`NLw+I8A)JK#XT*`d$~oF3WXHt z?B>Gx;kQXwB@+E7jyKp`&c*iVyPA*e+{@f*Q`aSi8Ba)s<-^2;LkO(GSNsq+ZGD?L zFvn~zLE<*UHtMLwrM>w{Xoo%Tc5%beav zW#WVPJw(3d1fgA=stD_-2Lo;&+7ylbo$r?X@q+ z_qEl-bkf5f6Rkc*!Vr zy!29N-3avJh+cPyG)aLdpoEzK`FipL;i%J=GSY1CHaq=`W)+>Vs45x_k%VU2Qj&=b z6v~*$ze+YsFQVw3i9XS&8-m5oFG<3RKC(nnfjtdCtidTMg`KuC!sOIBWPA()o8?JO z-v>yLmsLljS!CSFR6=x+Qi^!xdiY^)OS5Y*ehS`0*b%l+~t7G<3yw&##a! zP7Y76W$XfB!8QqzV%Xnpe%Ng45$o#0f8!LnkHt*C?vKm&@&R<-pPhTxMNX!L)4^y6 zwQ;~_rpWC5;3nArb!-hu$Y4bL*5!TmCXZYC)SDT9hA*bE7+yX0&NW_wtiiw#BJXwW zkaOKR^7>GF8GIG>vgG#`9oBnE%H!ceNH~u`BYDz~-{r1znHy(ULU$A#u(nC?S*lV~ zNoy*R6?esR>ABr!@c!Yidhb*c{}aK9o#wF5*QGu=xPTm)1PmO6uvoLKqtVzMfWuZZ zCx;2>8yRo#Q&kNbHy}#s7y7L}y`B4)vx@75?cM$_{5yET)%J>$(s#@E^eWN{kD&!< z7J(@+;K)e|+%;l>n{8pbFO!1*>Ud{@T0a1_;dJ=Ql5yU9Dw-lHe^ zy3{j}i6V>x6(QoNdLq2qGWtGcCZ1nNah2SbJP6{Hi7F)glamF0;R#${6s%bLSP_^L zjmm#ZRxuw``Q1>DKmvC@BqJY@PFGa555+($FxiGAeFIrRHMKO=WDOEXcdW)FN|?A> zEE(M^V&0N$J_LXNUbfiH{v%3{_MN;6@W#bjkF){rZ(1#Bmkt&$>ngP3QNabY`X;vjqrUK-jT`U8*7f z^$Ui*D&$PVAC%NXb_967F4)7b@0Wa^l2xhJ4Dt=~BDVk(ig}1`6ydLh!v@!!0|2w> ze8VuEsMJoBxW@z7jpSChUT>B154OJTTLljq>bd)JU-Z6z#GuhhU$TnTxR-3I_J|IoA=Zc=3_BGKYeqvD_Es{4+rWy`F;cGUbJPgy@J=EpB`NkoU zB24NrhFX&9D_tD>pqE;_961&qo)~q`Xyy+%in}9< zB_%b&EQ#kaH-F^`v`~baYFcs zpK!6VvNpERPYrnhqqO~3&vG}usiYKcHlttc$`d_>Nn&yeAZyGL{YNOSaY$8MQz;Zu zzyke}kA5(gXlXUE=Pu&0<)HcbiFn0-JsVoix@lgfVf6|rOBoR4cJOIYiJ@Xp+L_oKZn4XcW;>m4iHnRRS~W&N^)V4I6i?yt(FWw2ewo0!b@%Xb*1bHdV0Wb6GlIEED1Ek{Bx%!ID#KF*SAcxG*Te6vbY{NAPrG?rw94@UNtLMdT4M&0Ufb zD?YOHO2W)r9x()Z3f{B$P~Mu8Zd{w@P_4FS{3;{ly?3Nvwl91rO6^3QFza`GwA1g8 zu;<1=OGAGtxi$uH#^J1?*oACJizU9E-H^mTeo>RY|5Y z?9p$fMjGYXwbsp9-jIX|{A1Qiyk%W`o_a8xL3DF&>b zP1aE^0p{3E$pMVppNkah!cE~K-J3Wke+hgB2nQfwgglE4ZYAZtfLhmGasPwIW&gpd zr{bTYp~cOFEDNZ8Z76|x#c|_;^G(MSf{F%utRtyY?zw)k;Rz$KIE0@M1lR$lLjj-e z#z0aJQi??u7$=sIAZcV#Mnyu)9T04343{pqsHD53sHzIoPD!@_PR`U>H>M&sDpeBR zbo+4&0ey>>5>j-me8EJW(7(1SyZ^wf%AF@J5VSI3GwcZ^DZW$I5nEdPxs*TN>3Az( zoC-cJ%$Y{4uo9ISWvri2(iL0$k7vDDkY7&&lXEF^sY%s$97xvYrt_7U92pcV?zkMe z&1FJK3~uBT0ua7z{$H+*Ih1T)zw=|OMLg%$EN33M!b%8cjDMu}22zaZ7j(3%E#cnj zTA!O;pyT(Yhe(t}r9JxDZ}+-NR_-Tzcx(Ojm_7=Anc+nDeVx%`m-i)2e&-q*W~#EX z_6RGbLOxb7EUpF{dmp|0DX~@q$~Nv7kf;&klXV#ljbRUCL3YZ2)PB{|BZr3=x_?Wn^F)ves0%3SJiHRD*^(CsSvCUj=wiB62+;Qh$YBnr zbBcO;as>z(toDHxXGt0ih3 zNGQBPWFEUy!tWZ(1YePgJ2E41ye?_G(Wh{?MpT8`;A{IW2Ct^4<$z}Ygm+%an}l4D zF-R6yTc=|y)n+xD*9+r+0(Y8jQV;WSX=DjA%#65jI74GnA8Ka$n^RN9KH7x{UU*u{}umbUWY zXb{oKY30@}GHO=fX_!np9`C%Uh+j_8gVZ+-idfc!16B@^pcFzCS)-hx)D>S$-xV_} z4P6(P-QN=VRJw6dfBZUuFLnf#2AF&u&N2!W8gBLRyppz50)K4oNS^davmf6LX6_Bhjhpa!q`sfy*y?sB$14k_J3Mi9O0>b2Q@|R0rq*b-gmdA}HM~zi*jQBv zm?F?MuL=3X1R;d?o!AM2C!*ot^ z9QYKtYJkjD3~y7sO0M5ER>e%BIgjk%X6v5}>v%eglaVYe0~zFT_)b2DF@LsBcNh#p zWzr$bkgXm5^4`6bhqOyCt*ScWmumGsf@A%q|EnuVAGaNDmEiqWf^tkJr8AQ001367 zYL~{7w}gEB;V*-^i;xWLSZdXewm5R^OXUErVUk1wUZS$4Mkav<99*3;e1w^R0qGx~S*_uH#g7Iyaf zljFQg9c9J~zaxkR=$d{>9j9Wn)aT)xHJq}Y0HG|*X}4KC>0g`+gVNoKsS9kEvrP0{ z@iO<9{2T5mGrWy)Y0x?_hK>hA^Don=y?ed*mc~m0=_PM;1pKQ5{;iRSs_0wF@zH>m_5XYxh)2Q!*Kq}jiY z<&<4rOQxBK^_Os@bi`v#HbA*0mEM#T$=uTDZ+W#!z4k{uxBz?hpYXN6+J=^(R6eDV z5iv*)pAJZNFz{BK zWAq!7oV7$cx5YH@a3y;TbOXyLdx-aKNcg6eT8gV{s!~Q02vj#z*oIv=hN6GqXwbTE z3eDhYMXP3S#j%1#=zGT>k8UQ!%eQqc-|YipF1&%CJ!iem1a`$HV zOab<^Q4oN!ezh)vvq3>c#nJCg3DIZg&^PtusHU#JqLbDTG&+=eLNfWj?p&-E8wOk9+9 z3P0uh*}AI$l~Kq(zh=es5#ChwKU1Ui+DI>-?oW(p#l;byhrX?{noiI^#5D?-;A(zY z9m8X=T<-uXCq0qHVf)`dD@uQ4|3o&Jg321Hm6hRbqK{-jAQ5O1c~$iZ~b5<$Jeny!anKw?8Q`Cn*bli$3;ooHzzYy>5 z9lA11C)EEWqZEV~!#^no1TLdxl-Xh>G&I1fv{`=gy2*bnx`FNoM((=geW<}_uQ2h+%`HiV!P$V6GBi8ef@yYwk?W3@TVc^m zLUiiPwP>U4Td`%Ky7e@vAI=VZ?4y(uVlpS2oA;l%zAyv*(CM+~YmikulAu`2HBwS+ zCHkUsVi}tj(w8V|*i0sBbuTpf^t5|AJDITI%8GGog-{EWlxDS*+`MOcXsL&^%XOQH zpzl#@{9;(zTvXLK+18xK(tVhA?moyk0nsUxGQsTlY(n{T#xyb-c0l##c&_0kkQrE60AoRapj_n?0{Y{fR>@xG8%)q{B z5to2q@O{XqC+XfF)~bwL0)iI7rR%iz;i&2Y1}177R93ETw9sPx#P?E5n&ZhzNaE1X zF*EK|XXx?N3EzY^Q6N$sE8S&9Ga>8YiO1Ss*V{#Z9oSB07@k$BePB+6doeM9GccqO z6a6y-BTmL~Q6Du$DSdtZ@kViD$f(^P%bTbi=BWXjuXDu;#v)&(Cco$HFjhh13$6?c zdm1zPrPhx`IZHQBjo4ev*Xv@J@cH z1AiAVWPN?8ZpI-I|oct52dtvZ%8%v zPX*dqtPYo550?*pP057xH5sB;*k@fAlpgl$C|iWj*>TZ&^1C1gOJ1jnt(Pr_(F31= zs-qeFscfDfa(a3SoQUS;=XWoiv`{{qo3mL!YhL$no&xWnxj9w`&uW|>JqsSt62)6punpex)P0~BFNeLtrg3|p?y z?@}w4xo1&T6rp>*q^6ElDxULf_U;2$Gla|6VAjC(i7N2ZxFL2Q2xAo!SYHjw@>_F z|B%MDFTFqPWwSGBU0vN#qnZhs2-?B*q~2RxD$jY+b-O0gqC1H!6@5y_1`^;LT@DK~ z)A>z-*7Lc&cGR}%xd4a%<6q|78-sw6XN>)knIThv!?*>$iS~zIJgD-Lh$T1tT{mV@9EUdU}0LXmGOW1W*kI+{E6|QS1?+ zOOkD8kDf!Ykk<)ztwQ_OL{U19*ZSO#dkCV{R!^_%uQyWV-H-6JkOr%EYRLw#BlR)o zYj+KECm^KduqFo!9UF9;Wr3qVLu(oQGL6e&zaY8x124!$FRza6nqDtLq3*#JDQc-Xi#y{W|&uWE3!}2K{jm8nL$%sK|z_RRE z<(~`N+`F}QC;FA^EIc^bV}5Thl2d~6HGOqlIq3V_010IpcHiS0G{0Of*)fPmvky)( zi4>#s{glU&iMLUl#!h**X5a@&A>iZXzV7)LmT2S?t^D|4KEuzeU3jRzQhC++h^W*W zrFXlXDcEBg>6K}o>7Eklb!G~@5t*rBI4MD2ELn5=$DMyr2lp>K%NHRHGP>uCay}EOmXBb^Sfz*uE zYMiBK^K=XbtE1Y-+g@uD$wFPEOJ03HwJNmn)GeXWZ)Z%+$nD(bK~@{pt_KwR(Yl z=gWHCu-7`W9`%;zn6yz0#j#$(Bayym;t#j5w%&t97(1uXP^_|`q5f*A_R*Kdh&UWd zgjt2450)9*+ZMwwu$=a`UJj@6^~`J$hIE4kt)x_2cjEA|!ne-*Wwz+_atEyh_bFBm z$VkyEM?v71wJ3g(xEv|W5mD`3AXk~E!oZpw9sLqqH<`B!n=`H6o{U?9JUqS25Btki zu7`%e{Sq8nulrpiV0?F!h2{Mn_#^A;>A3?Nu&ZI$RzM9u90Dh|c+v7_Vkr7BNjV)D zy;}hGcm5G^n!N7Wts%D*q&FOAo)7R3zJ~A9(kbS*;SIz;5y+6=zi#Y{EO5eBd>6-N zea3}zlpM%)iPPVNQ_yLR1>MbNgT9;G7M6TT8^f@(pA#)-i7CSvZQ^c8${6;MB*fk^ zjo2sN?rzY@&1GT0uG#zDzM=Q^a@+&Y$a*)SIo+!tBYBl@;b#z+alA^Mq*N1L4rL`7 z7J>Kjf>M)q{;6h`mNAu|@C8T;*zh~YYlxulC<w_Wr{Q4!=`OJ?BPa^&>5<2gbMw-v#g%uiZ*ZD`;J zE_PDAnJW?5EPos{s(P$}@p^43l6js**Juec-=s1i@}FD_9>nOr(nGGRG#@!9sC=dH zGlV7O4}PmwjTcl{mGt$t)Vq_v3N{P7wKp#n)3gQKIf6RyL!!d5w!H=^t4h#fS{85tRk=YhbjCgc_U3<#|3#jk{M6K-|Z z@4DQHTv3|5m2VwD!dwXC!++5haByz=4q`Ox3@iCFVE*|tAt|ZZNQO|V7|D_D&FZQs zRn#X^f`Pf)_kq|LSXi&n0=M?|K2U{Z2jIx!LIRrmNmmIb$r)Pim^FLi&L$qSx3gspcrX<3HB_DrW|6PO+ z^d--DrUi93>YX8!(ZQC>CT5B27Ylt;!VvZS95LqDE1}5#d+qxpCSP6+?WRJedyT>_ zn$Ti#l#YgK^M;3@mc2Rep+@qK&2()&4wiVmZhhabaO>AIx~j|5i!`4fNP;*+4~BmZ zun-k@=44Ns*Ym4Yzm$T9&W4)SQ2$`pqASQ3Qx(B@@dA3q;z$yldAClFfq^kPH5Itq zDEN4H`l(t6h=t|-u7RnfogLfq&2-(9$qTstAn9;Ba>JqtwxxyU0)??ORnEm^JCSTaE?iAcm#A8zm33ECKBfX^IZt#3tI!Uk9FduAT^{ z8|h<}J6S<@r@b>(N+rMr*r+F^sYxu(TAoTFX)#GEa>Mob@BC#x)w`p_%ftLG^HQ9o zxIgLFh>wSDq`8|Hp4V3ytj*UJH~$i^*9soqEq^&!DL?Q;Zo7t38eUIV%P1!eYh%mH z7cH4=;&2|Ha%D%GBR~ywyO{>;4=Td5%rt(m%^5Mrv!n@zMBJC}0 zjz*GU(W|R93!P;PBhCg!kGLVYN!K6u_9!vTF%&nZ{2TtPpTuM~r_iLA-E&xsXp%3M zxlGlTk20$f9&mm3A><^W-(2)Fh4|%vt3c}sI@zRRDIPZ^+n(O`H$7J3L- zsUfH>xw`x}S2O2{w~5#D^T_hjn6S5apUJ!&{qWLcPh-2ruE(&Xri1|>&wJa%TCb=j z6vnD(QKQ?W>G^qgQT%wWX0Il}7Z>qcUQxnQC>wN#{M~+MisGv0_G>jgZi!JAYdM7< zQFD)6OF?A6f>1ofaj7T2+a_r>i{SYlG}>Ayl>8Ej_uL@x+?TqbhsC=omgNS^7*i(p z;lWL4w}{~N`LEt;?u4nO3ooBBL`$femueB4%KlU}ygD4`ik)32q3<_ydI5YIgd-t9$`R`~lW9^=sz@qTIm)`p4U$;y=opM?!>o6Z_j%8K!26VADl`<$i5CeT0S$uCU=%eCdH z=nEVAT$^EZ{moCcWkQ^zqo?PFw(X5w61oCyvlQNZ!rYsy*+1VT)?La0OA<_fax%)Bm{>^cN)61 zWQY@jU_PAo8~|M&o}QI9sjujj;;e}y`?e1bfFCj^m=CG)y4f*6pBS>+2>yE%N(7K4 zHaRH)jBooQNKHK&*|m;u?}6|vR;l1M5fRgDNl8hV3{58(1e~0lXy;$Nc#)7{%2D48 z9KPZFo`KJixU zv9WnpbvE--{OKF(1F^J5ds|yS!9vBmWv^IJgTKZs)5XdvrSU$;mG}xi8dWq3Sg&C- z0H$&>%F0;a;rxh*NXP)5XnMf3`so)q_-~Xe9N0nB+RXQYnIt(Sg{j7BFyRdfB~Y(u z%VJvsY^0&#sWSsW7BUtY4K5l7JG;tB&#KVb`8m+w^Lm^y6n!Xm#u9MZGxqXInNg}Xd$n}WD5Gn5+V-5&HInd1lbv5(f8#i zo)-Wc^eCl*!HWXZzKA!T8$A!s;GxH`ou8ktoL+-R(fzhx)Z*PG9<>?r~3kxJoce?CHWF*_Ja)^W&Ds3vsQ=5p68b3+H$QFY;yvKds z8vb}{Fn`tic=LSuaY@8@>*)5dA*3~`rJ2H0c%Ud+^)u?k#JHGUA@;+*)dSwe0@V@X z+Ak{{yNYyA+xaf)Wv^uNhds=tB^Oe3b1Vm%*>>dGTH|Ui8KWF}StdoJ4>h8jOm$7o z?Y%^SuV@bs=woF*M*hhuhb(>Q|E6+6w_ax_&c)MQGpW~ut7k{jnIu>GC-tIRbBdM&Iouw?6f;b0QKPJ6_)Y0^#$baR^)U5ghnU<^Shtfut}2 z7IDDqm|zw!5oOH&-DW=TgVD8D#uqMK4=lxrFJRLLYy!#9e6vV#c6Cks=0g^VJYO5q zx0V;n3Tz~SE4AAA{JdV{H1NnKMB3lq2c%~Fw+HZ5gHrgS(#5 zMikB07oYM?Ly9E#1| zebEp`4;tXtGT-*AcjexC%gD2R_=By}|Cv%CGAUDP(EQ7GQ-+&hYiF(e{=d`f3YIW% z={PnlR6TsKhI!n}!XQtN76Fbo!oP1#=kt7Ct2XrM9D-ra+uNJg6A|vY^}ONq2xb?! zafyhCDp*RaeVOKV!Evl)W7$_0vs4Hd_@e7ErpcexOIYdl5yKjkrdbyY-?=I?^alMG zz{@|li}@zjOb4_s*&=wi-%U=lC2%B#$_RA0vDyCprq~|O3_10aUOaQ!4}W+3cm;Vl zJ0xH6T)sul#lGucTU7A9x?>gJ=Koa1=cw~#d&ksH9z|T>(MJqzls4tpOx3B)!o)q* zviAzDkBbfGFtPMlWbQsQ3d0~%OIu36dV^d`dVOZK4+?;gf&yEoksIR( zP1h)Bu1W1g85lDt+)qsqV(?GqYpF^Gdac4=^`k+HzP09mzzNI_pmR!Upu}*MJI9n< z{8&GsJ!^`>89FKW1h*kMUF_*ZkP7rZohh}WU-hr_5J4y?JGH zs|^hfqgA8{yJL%$;)MQ~)KAkU=`V={opCuE#j-fx>^xHq`GZb`SFH$3wdl&_uY36J9&V??|@ zR!2x_IzD&Jesiqc#V(gEPJ3T8?g>2b!MzXgT>#oMhZMzM(Er?f`cI~A)cHJHrXJQ& zWbJK*UevsK*pUnxM;lnP2!bZ->Fq+W#nl(qVqk5dYA=8k}l7fsH{?@Z$6;OZB1`4q-@a zlrKGPbx3{f7bq*i&;4YO4;%K8$Zn?xy)Nku;U@48?`7TzGxGRJsr|a1b4gUw)T;23 zZL*|>YWCJ~Yrkht$dLAcBa(VHZDlim5kXn2p!om#Luc`HDyRSMl7=vyua{18Q*!U-|^&}*H z7upm-{%{nmpr8;3g&5SuMdT+5Z;2&Lk7|41NE}b=Xlpa%DHdx5VE-8zfqAEZ@d7lB zdYYT7RWVLcl{f7vZLSeqgIOS$d*XzS#6&jpgdkd|Ju9vNyk6t@` zOBV-5oB8!EZ~y94ze<&JsVhmpPW-&wlS0GnLoM6DS|CA|`A&g1Rd0S+4L`I&K4gJM zn^(0e!uxFd^`5viJI+LWI)z0N6yi@9Oc!uo=lESkP#K*~BO; zEa=|xe#Jl-HZs!SN%)YFpd8|XCRyYA@P(=&TNy>@tLY`>4-}>^UVPH(Hm_#2a*D$; zt@t~?N7dCCZz$yq<`uXiq$v11i>k|u1e)Xk{yT?6Vo&O%^e|4@W@WRtm+9XaRH|GAVm<_{9p z9-m}}y#MNDmE~5#C#*Uaf}{8QL@Xq3i!+Q~Cf}S%P;^|vb~pQd)nu)P`I_9F#?y0s z^CK*9Qk)=nZd_1iC>TcuH+6UoLauh{?6 zv))s*gBhb--`L9v%R>wKqCvg><0}g&u9)+DaGfc(V%yt?A`7CiFLCy~&3t#F@XLYQ zwIr&dU&Pqbx6})6a0h$k&44?MqJ>T!ovcIf)oR2mfzj?V*gJ>ab346jb1@v7+0Oe* z+m(l@CARS%rRhkak&cSP5r$2DNeik1; z-@;@iI4O3MU1Qz6;48uY-e>py3cRsU76N-d&Wv z?pYG>EaH8mH9fSAH0eeniwSWt4GnzO^Gkx?qD{nA0Nfd&O}($)@CuWdL1rsOlx9I@ z{jLz>Yig?b7iTosMRY1}_XA&;QDa{G&sN=^)lxZ`yR+(QE%dSyn7M4a6RpgM?_VCp zlFz7JVHd7T419n~HC;!lrJ!2*_TF)QMiPxiZy25C-<**mPXg0o0tlO7z(or*so5?Y z=7#BaftLnI5=#sxU@&RW14DOL&pk9K4w%@%m=4h2u`j=DZ72eaO-#PMr%-`W$%0Td z06Kt`UuM^oPoCsbTkr_Wf?^NnTC$#-1MfVIISM181g+2+HPyFZ90SP1n>TM#R~v{@ zf?$)f|Ma7xpg--K;$n`E9Y8cjmWoXHP$mRkq29 ze{q>+5cecCe8<&;zu#21E<3rIdMEXPB40BvhsjSOtA4V_R5-Iu?DVTVc{xd+C@j3H z$&|bOS1Gsaa3eHqm$&4F1pwf5ri;U*8EK(we-o}T2F#qeP5RD3dk2HS6kt?OXsO1b zPEHiEzTTQUXrt{HH8O*tyV?nP{KaAp@r=`Mg$VJd4@~7z0qv08rl@K_%SbH3J_BO0 z@gP1ul@Q6f?&Gg_A9;yxA~TIyKT*L2s#yE%ki1QNKIqhQ8xZHH{vEy%X{qPKXPkb1 zIIYsh9|o|pTj4D25P*)KUyR9IZ{=&dAgWQZ-F{_6l@M^-gf0fe8rn8L;0xP}i-ZSB zKzRyb))gOHemaRJ#b{hn(gqfuP(*$@Z+|$^6(E??M!H)^x~Y8bWa4=LiZ)iAP(9@L zoeIoh)xq)J8>P*6Xo9usvtMB#kkNx)Zy4bBl*3mQK<%Tj@Qb?pll&Hvp)5yKe$k@_d5B$axc49Pti0S zz12XuJ(+XMGeU36)b}-7f&VC!m8<5t#jhkNWh+nKc{ra0D@D_r7HfP{LoAh5k2?-J zh1N%njo5*dHiB}Pyq)X2*4l)sUE98B1$sKS^Y}!~MOhd?iZXVgUFL~czc)Eq3nIw> z^+$K5&an-r5dpsFnYWNZk{MdL9BT3;%dL@jpaPxx1^1obGmu!-X(jnvI8uMuY;WPU z&%^I9Gf6;IZtWa=ysYv3Kq3i_G!t-TusEJHyAK`wZ`fA7q<;wpZ@pUUj@@&^bR_56 z_v#o{t6WO4$QEDz@~_heRhp1~>5WGn;o;#inZL+?iRz3gxB&t*-?~33p71tELj=YT*15 zUl^C^rQCE{9?MG^xYpm++KN1e>NN2L7N;dsKHTHXxWrXk z=FKMF0}Rl5u>I|?D)je?Xw9{O%ZEl9G+?xu$&WCL!(yGyj~Lkm8BK}z_nn=c`A3LA z`p{Qffq^-}Kp!JxJqSEv%89-nGhwtqu`J$4zFM7e zG=tw>bc7hwH7W>*l?)^=OjEQNF?KGqZMJ)TE^sZhzGYG7QM}{(G177z`kkhyp${G?md zl!X(c)q5~vdW|kug-f<8L(2<3_)2&(7v+Y+$@*e-ZZ>_VTCGB}(FsY?4S!%#VG0qe zZ|ZR@F?%#VbC83HyeTHfS?RnyJB zZObDzb&I`#Ag@?;-F8lqCiiw$N1>G@F4v3&E7d@JfYO40T&7nC)zPx#j2-+3kUa!|76J?dP<{Z#e42Mb;Y2Cb zU^{iJ${}AXU^Av6O-sU5qf)D`Kb_bbt7XQXW58u0Cnw(o*>sD7+_I(UN-?F>G4;^F zu^efD2>^$Ll7{&IQG8N)9UhS4mESY>o$_%*Y(uS%kT%1e`4Wmd#H{WFoa*s8>b%*I zh*x1TySt)ScY&7)1RvLm6&f_;ZJ_5IVp)Z(fV{7)j0UM_#S1OI0F{m)6AFdyO5}`b z_>)<^@!?jaCd7aeGidK}V|0PQyhJ=CwW7&<9cLacF6jWFn^k8CQ zg0eP6a1Mv>2;KP!-WXL%=q^;3xj4g~1i8;)XAVhC?&SJzsw{YRB8V))SW$sM0_XjD z^*a5CiNW(|1T^k#0I|?9EkNVZ>cm%Q20W-jc3y=}J=ORn;n!NT_ixxnWv3Z3e7IAM zN_fmMo7k@HH0GK$C2$(#NHH7qpXrbELYJj$>xlIUoILVR5M|h9YF!6VW5U?=AVRJU zH@vF9^5QfEq$qN6WUONB|$sbUx8r<9UB3six_4ldXHa4-ss`A(kH{LEblU)j%OgyVI*q zjMJcdrs;7~B-e)mPTNzPA!!jFJ+fJ8)5LlJW5-y!UAn9?A0g}Q>$9-2$pe(Lt-9j% zy$_e7I=Ae%j^eUvmptIO2qLZ^w?GvJeg22Qs55PFjSFARw&9esegNUK_^Fv4;1>eA zR)#cUyBn`!X}}2w9e=h0fd-w*m~a&f zE)*z39!=wsRs0;L#-&EAlu^%y$v@N#%R8KGsXJY$jnQ=5NcMY^VcO*w`;ST^e_!(l zawM|M?3?WAD?aEB@T%6BO4JmG|L;TKYGnYn9@cun-FgyVegp!nfoYE=P5MC|5b6<< zlhY>-PMx1-0*1%Z5)HH{bL|iVz_tv0s)Bk`rVhR3=l3|bnndv4x)(|cAxUXUauZDK zNG6qTOl-3O;y%E<*rqsCsKl<{ZBtOvCg0878=!QouhW2Uz$%P|8=s>DVD}$Wa8-`i zbsEGZiK(!I4@E;(VrfkIW8<(k0@c>(AP$j8A!j7cuS(9Zp$^a z0Aktmn-Xx=Gj+n_Af#7C^uQ3cp;?@ctx_nXQ87J7MV}?;qy27K9WWShUkN0*TEX(| z2VHH$>9D}|-d;ER55CzlIw`WZ5I4F@{PIc#!+cKVM_m+pj4Q#K- zPv2QF%VuN@OQvSKnPYpUJ?9iVDc6RngldSAwU*F49vdsyA1XnE|Gwz2U!0Nd+&&}gG+`JVL&0dwCJSRMeM3$!QP zM(f^y*aDoC13(dhKnZY>1(lVRtgiB@`R))al@HHB8=pd#Z!z6MYCR8)@WHl|u(Rq* z5)PW)6D1;|YWga`>G#7m`*q@>&g&pmA(xp#AL#B(kW%kU6vlxKSSGPvLSBGFkDQU( z)8&HZ6h47wHg3Zh{Rw`)IaJ8(b(2=f@`t);=3k33yFk|HHk?)_LlNB=F@a4<9%iDb z^l|u4cG>B>jXx|h4Xr9_;oP#8b4Gb`O+Zxjbj3VhKXvcz?FD1vM=)RcK3@86R`!vt zfvB-8kiQCUZlJGuVd$|g_3w6u^;aA_33>2dt;ArM&QjYV&^ zC5nMQnaYLjp)oWpS6-FcB{ANB3PFZljZx4%h+|V74~^zy73n@oIr7T!GotA~e;R$Q zhG{=uvCKwXd{e2KJ1tLkFd4WkrVIJMLZMMzQPKV+{Cf%BX%LX@zFz~|B0E|t{Qp{j z{r!Jg!DOq@rUMgudu%}1fI;->R16vc_r0|z-B}?We2AJ{enA1B`w3050#an3It;N1 zVo+}7Co#Jsn3A8h>j6RNfsoz^RQT+OkJNw9mU$}_)So-r;y1X1o~lT~4#AwCq|m!^ zUBX)rZ09-S-mlQHQe$N8NR0ch1-8g6zM+u~pVJRT+cV7LA>dB4wJHjy4O`7*L80l| zWFnBE`dGD(=!fwBs`-iuH{DO+@w%c6(VP7)zv2nxiuY+`CD#J@ZSw#32B_`> zVO87xd>ewkdV-b3CABH z&YDtM9V#t)T3M2WXJv@?(nvb}YTUy(jL~bZ;w<5CTB$$AlKE7-B{A63% zH^sGijTDeL+~rU7y6*|Sub3cN-{B=}{eG@?sQmG!-0868(h)SifNTeZVh)o^smcf9 z6eNKg4x{gNMT*$~Wg=#`eyIt_6sHn$tt4mzn0(p=~CsvU&4NyV_|N#*myD z?3D9qP8W?{u6Vo}MnpgCtC81M?5qcXVrQw1k>=h=@<*-K)ARD|(zdcDN04!Vyw531+q9W4(0cyp`J0apA zQ3l*>ps4`NF$@mqGQeaz4Ox8N#ST_!r&KHMm_TeEcy+WBG=#$E=E4s!jTJxlBcZM6 zBo9U@|1d^ijg$cl8}P1N^5T}&%$?WuE`wSSfraeIz8uRKhJ@P~(0VO>!#DGSBpQ6(rfV-(%Eai&kX8yF!s%yKxF0iBDs_Ay)GA=l3QnaU+)`kMhwGH~6Iuk~iC{NxhUgayyTmc50BSfKO+_?&bk5GFnBl)Hht z4^rF)^U)IA1<8P5rit3m&^=d-HWznIpmZ77{tCPgyyjVm?K6%JQc#eYi{q7a_%_L2hCe7{e1pg`_4g zUui(&{tOK{fC8%+@7F%bEhs->(-5;qL@bdP2fpE3A)&Ij91IENW&c%yaAXmyR=SX? zTfFK@Tzgh=>EAy)DyH#da`U6HkOykJU)J6ySknmdBqz|Nsn~mGAymYix=Hu4tD$Rv zX}khjSm33RrSjn$QX@39hMC8BOjE8d&qas{m!NBAhVamR#Pe#EpsnA)b+Lx!`~w3n z6;8;})mo$-KL=&z2&a&c!DV0w$oT`JC5R0?B?1sC<*F4m)`|;f0{!&|txR@|neo7n zZX5-to0+qnB*S~_X6}O^0+}j$#+hoM3T)2AIBB-EWoyUD<$>d*hnbQgfYO?_BdT0~NViXW(33{lt-okgyo>V^M`9j~I(G1BeMd?#@n#Nj0z~iCNs;v(4oy{696wgel zXSHdT*cA?B{9kJla&h4Tq;~Vm4?I98M@QMK+7~7w%+bk35S4j(I`=Jf=gjS-zBt|Yz#s=Ut6Nga^^W! zd;am^3fp5^(k)!c8^;n--_SMmE=4Zx7Z^Gam()uZLaj zNrID;U&jp47?vl3oi1;$lh`K-xoreqc^uf`0-82ay4a4j<0u8X-HYqGnwphHTLW|i zfImzTHd+9se1?I)#>3u$1>K1)kRT?lNz#FOPxq$Z05x$E!ZMF<;% z@~pmu$-79GrrXNHAV4uT#-)f#3sAQIZX@kC+H5KXM63lvi2vj%roIjZzMl?ImM0wO zP~Kn?7LCC|>hCs1nQvsNB}kCWXGBsn$-|$<6Io3NcXfHe#~p|f$Tm2(8dhcg=B5N- z=>Pk`av*34U|Ls$ZFlk+ZmXz152u;23UmOeT5wwn2g+Si0Ld|m+o%EYZp51Lyvrhz zcwSuYgvFC^YQOOu5e7-l%q$;%g)pT0Lz}aAfk9~!=-($-d4#Vdv{5d%?*0pEjVaL&2F5;rjj{5TL^`NE&m?`x)s3m21h+?$LlINOS? zVm^4!y?M?^u#pyqEt9>87&GYX+i8d^$5j!#OMrg1?zrx#vruC7=e!S9h1>b*GdUmz zU!KyZPnj5I)$;;;42T4g$=tr)9=5ePS5YJm9#3RG;DXFmQqp2O8!;eB1=;}$T(qe& zOnulBwF5*(-#UyJ=O{r^(#|9!=q&Sf%(6y%*>&xZr~NqB`4PsaMeBLhH?eK*w=R0q zoMYAA0|*hB1F$x74TUtrR5!DDVL807AjZnezXJ~uP1*E9Sal_Dklm;RMivo*bprh=Gtm^BL;&ffJ@ zV9YQ3Dfc^KNK}RN46j9+!EuN*b378n=#%u!(bkf4s zqNi@aDt3g5HXI!3F-jBL8^Rra}U=OsRc)&nHqw{|K9B?3^q*I?$x zp-{@T8jZNv8`SYnywamPgP7beNR6QBH`zPVh(KN>4}RTWJQA*)Dy)F@Y}DiCz>hHlTqmBNvN3 zPxZ=*C3i_zrd*~a9-p-jUu_)|zRkkqL*U5t&$i-_3^7$mCZ$-tjMm@p;pChoeIOg2 zQNYeI0J|CRmrpdhF*}$V?s3u|Y|RJ*v*7?lVszC>Aa^dK4||iR1$L_RS^M_umZ9Iw zQi0+3B&&m2u$fDt?#&!WGv$wULEBN5+y&jFj$~zH1S;Z$P><>xF^)ye=!MkdQS;p6|U+ZY{E`q(V>1klfw6miB9LU;^ zZto!mX|wDYK(l86qkayCWPnd=m6Ph4e6bnQe#N=2M>9U!4jE8>0Tx+uUK77+#YFP| z_3M{4zh67guW?zNuY1`ttBg)%`4&L;xN__U3ZM3Wr@7a%hGQM^SUIG0+5^7kixX3! zEZmgEsU_YY&8zd6_EyBbldp8qBPcEHQ+JuRtX=m!uK;Dza}3 zVlXs_H~0jMv%(Yn^Q9Z{Eq4jAt7y?GIi^Tl(1pzDPb)N*p<7$S3kdxnPosVpZ}x$g z>t&B;{Gl`))GXS<`aKa+L?4~`*7xEnGemCZNkT9=P8$Z91nSP#)WsWk+Np1V-XpX` z0?Y{xB>DOIU`W_oYDq1~@CXII_CROpK=PEnxO9Kus@2>j7&~y3Lw^6=&gV3>c~j_c z1(d*?ni_}>_M8JrhWCXI=u@8faTSz5AUnPU8fI=}Atfy>TQC9C)&_AXwaI|GOyh^3 zWsYc_^mx$3%Q>8^%K|n0NrG~OfeWbdcm4i+sr1Y1&lw#EHtaX^N7vGcapsfK(mpvl z#sGn}HIrYvsUby7Y4pxC!~kek!OO|qYb)vfpLU3nSvE)gE)tE$_Qw#cs-XF=r%v8G z6=tykL~Zu0O+w|7oK$>84QDo}4KUS;N(iNABGs<%rz{OF<~E7A9>XqxKof1b}8C~y-M0*C#M`@N1$ zAh2U+XJ_W-)*O#Do^|sAe+bZQP-nw`S$u;)R+BaKbL#7p^;+G*?fM9iHxYd_LXD(h z;Fla2`1r8>#@_#?F(Cc^{$8!n@yV>4%#q5iKC+VZL#fq5r0P$Zs2--fhZC#46NuY} zzR8c-A9FTL=SN(-+Lg09H7gUDry8w#8WRnWLhu1qb~m}cRzGITaeQzB?D3y>8e;0r z%CsxGU4GxQgxxS_F*9$Ta4WTbb*AHpO3di8wQOF$tf7zk_`b?Gx_HAM!*GY7FCmdd zOCrIe7kY4hb2luyC%q(74M#vNqs7dN>h*_x74x53uv+QT4Iq9Rr8mx<L)12|>@AOJ=c(%CZL;sX>eFkYGOooWFUIY=tw&19@s znq&u9Ns1W@5UdDh16|3)gfLmWp}#*d4ds72n2?G+fU+msf{}q0_)stf0rT8PfYvl` zZe$b)_Vy!i9U}iXNC54P^yP0rRsi&$$w?r{;Naw3y}XfWV2JGn#4uIb-e13D2eZNF zg&6_rlv)cXCNgG>H~?hm>FJa78_UblXWibCkTBiRgW+pcxg88((m)soF606mLU8HA zeL?WZgaFu}5C{aUTF`taX9XOm$i|^G0{7lEr~D)UuD%cXILQtoa)6Y|lr_Q# zA3Qj4v#TE{fae_%5n=Y(qpWk=^fR@#hOS3} z(Au99OxmAZDOqtB+`vNg`MKq4m#4@$6HD$;{3V z22wb502wzvpVp4i^c%3HJ)5|F)2SFd$FKI+j4mL7dL> z^Mm{Q^x1d&7Vw$Ib}^txdL?-E9%Q=N!8k#D-aKI09(m8UUPX0Wq51>LUX{bLpx{Y- z9K8<{4JFu3MBI*lA1sbWy7!?KNXyG_5BX>Kq0*;cLg+EyKSeerWX=m-BXz>+=X`t9 ze9C)p89yr1GHJCHb$9Z|uC>)*m7DnI=d>3n%=k|&aV?()hCD)-gnBiddWPNIH@Q`+ zLR&Z_g{R6FI*t;<^IYOawR{e&%74tzit&S+UP(tshxIQm`hZxT{kY`L<&6_>CIBu$ zm0D<(g65~__^3dc2JDO#M|?3Q;7m~9gDpmiiJXK6>1{56LO~c~gm@k$c|rHe&jvg2 z!UrGevU6^o!Y7V6B;fl|cgJtk<$lA~K1?DC+rV8nff8~t#|UY!7i0Zz@pKM#Z8wU zmvCkiY#H#~SdH;u{Heo0@x?4xi(UT)`6+ldlz9Q;N~%c=*cosCqyK{ggtEI9Gf0t| z{a2Gd`!~8sW$((>m9H%=ykMh%qVo(LJl5eFke5#B#Y2Pvh%phOL5GL-phX1&hpBsP zV9Lc!)MH$&%y;c87=qb-4Rk`J zA6cFWYhc3Z;gU>F-D13{Ot9=otGm$yg%mR-4Gjt2OmjoS&p>UVriKHY#EYTY38}f% z;GYJe4QHO*am&F$@YaAC7;|*040!LCoUg>uF$wz zb7!e`xpXcY7!wt|VF@Wv=I4eqKCSkAqH*!76}N*FD+1ssAh5H56QG&h*D7*yBVX=S z_wf>TqFfUs&G9$TmdvctWqU6xC97fYlg8;=M3(cTnYR@(J@9bKX18`5zOrT&(+|hK zQ8oa!lvBdEkFwrk=SuqKJEvfWiL#;)5E8;~w}ceHu&LGkAh)n^v&<|J5%vT~M!`Zi zwL*lmQp)^t0f0z60a~T8EpV3|pH~KCT4ejB0B+DpFOz@#?ymKBLuR*TU3!D|jGM#O zIG|Yk@9zC^J~JVp$b1jbOE+DY5sr_Y-r`BqjzDSb zhhV_{?dw_a{sCo&{JwmZE5pxkrcCP>gu|xi$8ahWiIanm4B6{5~nG zcKSzKmxwRp^tM$&8*?xXy%L?n)1#Oqcw>o))1rzNn z#*2J!^`PsVBzxDQt}AaI!omp76w$R=^_>&G9X~E}MHA3uV{kn-4hV+QHo4)b(b!I& zru@Xh)8J(7Jvfu#1=&K;DEWRtIRh`WL~j#@^uEy0x}*>Nlb8GXiPg-EKRL1h`yIvZ z)e%3AnH8$35%>ea)SM#W2{36!CbhJ)Gh1r&>drk1!35Vd$Q}rZHUQlbsKtK+JVAO4 zDfEnadU_hz#Klm5(G{INSN3~RIZH>liIrmoiEhgO`zw9tyezV0p4Q~=%e;n$?;aS2 zQ>`*IjlU$BMi;cGTh@YHf!D8<)jy9t0z|yJq$T zk$&uCWzkk8hTI!1YHVzkRp`8OQKFxaZ(2oSmw;z+FEiws%ft@2)I-XM?+^0ahCr#- z{J%iAxAD)BK|NVzGs{R6dmIznOF&-PeoYG`D+N<@phf~dGP~2IGNDO8#tr`NCSAr5 zOwhtcTCxuTqJiE_JX!OVs)hzQ!Yt~7KLW`h1kxu*#kJ_vx(HB`HRtBS`(f`z=GM3U z;YVLNY(w-N5Al)SeVI?y`)?%!h#VCFx3|BOmVUQJ$<2w{Z4U|R+|A+*ob)}mui!SpULVC32!LT z|8W6I+zdVK=o<-FUj2O&bghf~Bid@SCR zsR1|O+!U>E_~KcyDn-o0ZAH1%?ChL*xk0&)IWM%vk)C`h)IIBIn|tCdfPq!c!!9O* z8Vdd}#|2!ud0DN+2JBG(qA1g}-qfJ?miSYfE@mBUc~)o5fAG!RV1=Bi-4~Q-f{88*|880p=@85bPHjc_d(I7xyLzz6e^W}<# zA=Dox*T#d~5VU)lcy_J$n%!)oX*k~!^JC8r#jYcn__+QvM;+XldDr{gZ{F7ari=tJ4xT5V^+xPwvr1iPjGz&wb z?r6`Jn=$HkwCDW!O4_Po?hi#GTg+|6@0{rcw5eirCaS;ocqRP&TLd<`3sU>YLR>b` z>sV6LJ%Wu`&4}k-yCfh zuhE|+8t3O30+uSZM@GPXS8qs{8uu{$4BHOci{%9w-elTZ9v8>+`dO8=%kR8TkjbSv zEev6tkV3K$VUMZq(N~1kw$E-l29Gl6%cemG=EZ zIt^WTe)nK&C-V-g_ZBCEy7=>d1n?S+_R&}sBLA&}ny|aRLfoJg=6kS?huOUz{*hka zcn>kW(K;@n`vy~w*?eh2=;X}vX1L*ZoiLw-viuUiw!()>yeG0@yUph+nyI|D>57cu zrLy)`I!fEFUYDzV-SdJ1pzvB$FuEmrK3|%ev>Ssk*(8p=Z!58TGRhW z5|wnoyf$xhR-tIp?DSBB0NDEgPr zccICRULG%;N+lmJqZm%_qrwZL6NB+>ZklNz>aIEjTNsrU(Ym{q*w zdhtsC+UmnBDL7cIo^0@xIAD08ImBL77&q zxn}2P?S|uO4+rdJAMvGEDU@9>}{<#XZQq}GAXbI zDFCQ*3y@}ncS?7k2r8OGE#8H4MIkPLv6)Pm)Q%b*SFG=Re=P;h<$ zPW8c80aqem;?+_CLczq`-0JsRvme2I{TF_TZOI&e?3RryuBMMdsA-tHle*W0(`I8$ zE@^6Tk?S45QZ0MDyeupD%inRqju!078-riv8+*P5IXTx>w z8=SqcAlJqsz-GmNx}ij_$BMwjWR(Lwi3OM4e$v#QU%cYis>17_U8jXS-gL8Se4l&%o}rD#$pQnVHGs zL24@1_xn(S)>?15L2BjjS`|azg=R#keFWS zpFPp~WlLh-5@pFR!|1SbxoL&otN6Lh?#*6oeAIl1pWs4yOIO5QHE&#%rF2nWGz0$F z)D~NGP@r4x7{1x(%2J2?h6*F$8tTRYhirk%wtzU03x1QZvp|b4)!kH6gvG`5Jc(|Fa$=p3CNFwxW;cUS*#o5fxums_aaDDU;aGt)7El;{^2i z3d59Gj=XUq(`jl33Xaruo@I@{L%ttBaf8`G>L}OV*o?8BKugN=S_&_MTz%AaZZz97 zCXV!S#5{K^DW{uJx(6mEvp*!W{A)R7hbMjxN>NBWlFZYjm7AWP3h!PU5`Jv$JRcN% z$iGpdgUe8|-m{PwUE6!~irL{l7i1ad;L5)_M~^Y-=*Ay&Y|*W&ALDtb7AI?_pe}~I zH1FKjGcXXMRL}?JQ3;Y1ifYLK7}?$3E&Ri$oze?jn^^j^wp5=0uwiOuCU%Dc%tGmP zo2vFf0cp=qJoatP0D#ppDfBdZ6eZA-8g_z6s|h8_5C zx|_#-6ZzjVWIr=XI9!M}D~#xM>3QQ7DTk||JtzDbF0h zVd6V(;H{4)&Q=Unu2Hk7F(*xO-`4p+z9v%$!~3fW8PJhke+})9>p0&GYIk?&Wo08j zLPdyP??6<^KYQl3PtmN&G0ui3;B5t&~#1B&bpkAaK-W5$C!Pea06kB zDa-->@L;Sb1-9O5007|y#!?`>JBNA*`Gx7Y=Pe-RxxnligZG1+|@yG=7z)hzygSn=VvJ6}stpzE0X zFITEA&aicXnv4FVBpwx=P5A)AoMr>@%Mr54%jfQlL73)NZ>X6^Co)UQW{qEN+b{$+ z@Qn+|<@+=k6}Dn_(eIRUw^p@a0)R2WO4_Lw0C>eEC20xbwX1w3(8&J~L%Gfzzw7lD z_h#KS0=2%1Kn|84Apan5hM&fGK1g*mArtgXPs5B5&;*zUe_^2|h&ch{H#zuJWa6hu zc3@EuZU+Mh5`e9S!CN!86^N9i5Eje^@(!oIq5rwSIW_BoZ8bVq4MRt+A@>w^)CH?b@!dx@FT^v#vsM)q@KM^v*G89BdW*_pW_trEeQ7X!U3*rK z1d|TNG*$5ZlG`R>^!4jC?f*mMrhyCB?X&Z(0CNI6E3gqX z`cNfSzm6KK@(qrj#yJKd%XP|PeC25&5zvj>&G@!Pl_?M01V8%t+|=2aG_L)T#Il)^ z1h&IqI%&u(2}&@4RZB}!MVOlBP#3r0X;_k0{CyIcGH>hf3bOYOOPu!T3JgJ>HT|2#xH*|Lfg<9Z6`g4A8%jQ^WIwSG{nphc2O63#lSi)i8W6} zvGG+NYfx67MdjB?JpX94t4QAw+X5) z2BwW9u*rtlu}v3RO|r9uyM8+LRx0$xP%)dMc_Z=uOKu4;0$XA(%Fsu`FVx~)$+MHs5&MR}lM$^UJT|yn-`LNQ&My5n_cDB0R;patu)?d<0l)W6bHxs z8+uf_%%4iY4^dXa&(WF|!s{=WusZkt6}I%K7(iRGKDx9Xk7UW5Y`?%Sl;Tg!>Dwa2 zgBf)X-tou^Iv;Us97bFHqneu=rX6c)BOG^`+pF(;x- zG!v!i5?4;L)Sq!AGjg9}*MG0P|Jb{|`}3pF(GMR?ojH2#Hd(}%j9AZLX}^-N-=$25 zP@$h`1E&ud^MZjvKf4^pJ6=(C0<+#o93U>Wnymywr6@e6EY9BTu~cBgXba04A7{r1 z4+c=vFUz-&K-6Cfy%uPi%cJosD#Ow%#RTf!_264=s-aJBr2Qtx5zA?4NHGkZXa5ih zrEYS?2^Ag+c6Kbfm;j!``H}vEVMJv00Aq&CC3h-oiWGDtVVuZa*6*M;+?B0x- z?j*%De%%Q_CSvd1;%ni6SNA$ypxQNOH~4eK<;>Siwr(bq}oHKY||&;0Z#D@E&S z^OwkYQQ^bV>`ZKc{mR`0H~}J_@|e8bvH2hpU@d7-3I&0&k(R#_+KVh~@R}~yHq?PMqS9*sbOs9^eTbI9z`zF@#>$Th7;DxJ6Pwp8-!a76L_fkxy zTVIFM>aT@{ag^mFYg{KYH(BYo)XNx43#i?4Y}w%UZCs zK~gfHM*OYKzN0_g8q1E4f4aE12(mbEne_Qh$Z5d*9zZpSdPA$sCl^$hO92mlDU5pH z0xykDb>|sRE&o|=Hb4|WXoMC=J?=}&@1l{*`{xZY2>_k|oy-`(tCOQjxsQl{D6Xl# zuL^x{F^?Pu}-$)jpU3B(6pgFYP*~<%^ zJ_Vxf+6cth)s z=Y>>_pMWUqTdd|kNfWdGB9qw{eFSKrHbXbmO@L%y;ati0#NhBB`iZT^EO|tAN@s7ph!PS}DtEVW(QLk%b1&xct>swT{7DURru zdG)iYCcQ0z$21W2;!mulF@-uu@7sJSq1`NY3o35h#acg}j~wLe^=?UK@&_i5BnT=r zqmph*7ohNFUX3+^G^_8tMBHuuP$9QxFoTLW0FL5`sqMDmHVZT|YGflY5hGImqJ=38g;q|d!&*Dq( zZxz>n84F=p7SoiU91oquwVJ?bH6cY-(iUp(hg4r5nh`Z5h7qL#uBJWuh4$MU)k=L4 z)8+JhK6#U^Q)VfqS~D%6PKsO0%=ERp^M)cNGJP*Arma};79M7zYP02uHMAU>v1$4) zwY-z>J#BsKviaKY=FzyvGMNzfm#Pm2sqeGoPxpr_tD33<`P3@~ncj`-GPc>(mb9$W z`qnOt7j>4fE*=%F6pmr))XW6!+=|Zw!AgI3W(_QyRM;Bn^;vgRtRi#2 zd-*-ZrO)40DRtuF%x(UIlV8aQ_Q1=BKXB3IC58RP;C1q?qRM#Fgd&6a9y5j!-@_p~ z#iY~U8jXYwFk8fr9+h#r#^L?%S!1EpH8cQ#S3%$$yHumH@95G5AA#L%6baqY+*}@Q zDcr9s6pCR>Ic1jbd6yWrf>7J>XRUYsLh?l};)H-PK}m#>sys5H3UCtO>dQ@g_OoO{ zH?G2Ke3ho`GsergGzxXv*iBRgZU~^0D5|KiF~BrGy_Ifs3yU_2`SMfk|GLyztp9bX z@u)%@Ci>eu`b;as>X_QzZ@X%$PQ60D{Xxlylh643^S+Vix6lAyWotyK!!AB zm7+g*kZz<@v5+lt?`kVy+X=6UP-6ogkAjRJbpcFS8g`&GX;%W3R#8ceu1bg|Vk$tU zNMLzBPfC7hI;pd9>UGg}$LX-|pD4>YY@#4E2uT-|nm5i8u2xgc_k~9eE)>=Dg6&ZGSqa3INs*1> zV_rF)Tsb{QXF5RLoWwl&E>}#5O3$XyRvG?fr?9!1__E))8*9!y%kol!h)>|N`t;B( z55v=pwXv8k&cU7=9n+hbe2cT{u~Cp3hx=o|Ji8yEblR+btFly!F=5U*B9`ouiI9-J$P&cqhlIXK86&2yMZ< zK5*qJNRXs8RAk@wzH}qL-+11V!BBUVaJ#F)D0uT?d3^?FgnOzwAbtDDa7-rdpF@Xl z%-^l{-sStj19l(Aiw|;!g*TpS&Ea0Ydse#(1Ot~DWXsM-%eM=9ze?SeFAVC~h?ehc zi2aS((j57w-MJj@PTvn7wOhqYEc_oAAk|^%2gEw{gWE^4cvUpP?6@(%Ll=Z>6hAD` z3IaF{z$dbfUlGt~0e2OYL~j=lfks}HmSgR5FhbP056_+pEf`FrWGIhxw(4>nCKYQr zQ$(H_nS6L&efpN&&o~^ncAg%L(7bW&U+}l^0t+WJv4m*ZM#_-)xKX4b-%^~vW(a{x zBVN~(e%Kf`GqMk|sa(s8TBC}xd^2oj^(C61(l2*R48M1Pv?Tw?(b3q6o<{ijEU5PQ zZ>uGCj*gi0d`xEL8ysL=4t^_9$%HMpD8J|DJHv{yb2}<4>%8XA zh8*S^kA{6OHgzoW=`-c2|v7T4)cT84JiaV^t+xyJD4D&YQ|L95l{ z9S_cb9?NhPAy@U|?2hVK-Z9IynD^H!rMJVcj{4gUb$oA1ZIJPi+Ag9x*7zRQNSvPC z$NM9PU?f%^J-&o=+&!g$FKgTRml4j&5`9ZIHy1Knic@P_enyI@o6?pA83Z%Tap; zjqt^Lhd$YrOE|TRlf#Aeus-5z{n8YIaB;8oUjt`0JI&|2vMh@!qO6be#bn5u2M`Ii z1KqC@q$y=)!JF#8D7S?*gu;%R_HG5H)x+)&8FRH_AbsXg-#0J{n0ZQPmi+aaSKG84 z`}Ff7H7FXW&+j$~Z5M02=1|gHX?w|E2@SZ-R<>5XA@3r4vzJ%O6{B?aQf)4douSLKs9@_`nCJF^uy=cDOlFWs>mEI!@Ix}5)48qV{&J&I6+@%zG2 zk@tq1ZsWFw@56m|mH(aep}uZ+vq%W@S{uBFY2HM03sszbEMwGKP_3|{vw!2t zaR3xG@EF3TYY4~G^t6Jzl}O7Gema+6=MrP!GP)JpPw)^T_EM(u=r+uf4}9YAh~boD zKicCeG%++haWxjN&HOJLg}V3uolz@tb2Ie7H z{Q5uxI)BFoGoWJRu@F==1&=MaMM9mFV~}AKOK`??PQj6xS9137g@E-G95oy(#oe95 zP4LyW4-X047i;J-_Lqn0m^ylnK-obxNA=7cntE3$*e!E&{qI?Z=_KxH#RG%APAmQc zH(R2e`8J>X=Vks=Zdl90Rc*)hb2@?`Rq5Q2iVFEG-&H5F_<=9SCa{!|eOPPwdG`Ij zSEhSy%+v5j{$W(uk_37vGno^lqCLG;$c4wKh%={Qkm|*LTi<>`~$ZdiPmC6r;E3UGOA+Y=Bn5HlrKPT zZB%fLc#}B>T`Jdc4;_sA_c0$mG&@uU<5g-8Gaxhndl7`-#eM-?iC7XLc@YsLFsoyt z)2N`I-crO4#i3JI)YBWeyIdXv-g+OYRlP6al+*Dh>@s}_q}0@apTloK0(zifXfWct<=fNC z7ml|h0fusENyKE!BXqCszx6j}^tksrH8!e0b=ce4)>D--=NnCRfUAv)bel)|d@z^2P{Ov-i=8WXpFW}@fCU{4| zRsMhm?!vW3Ey2;wRFc0hE;~kA9PUZ}wA6&gQS> zi@SGYt3ABMHP8_XzeiH|mzVwUIsYWP=`w}VY(sl$-t^L^?do^OaTjUldB06_@FnMu~UeBG3CUMyjT&2;;Bd=xD6VWrV?ktrs$hOwL>HNB)T>L<& zHzJUt@FN1I986lKE7PnFI^Ua(qn^nCr{0G|AoBIp%y$(#hCR=;FDBW?-F}eW1Ytgb)i^E$bq+kra>dO= zLk`T0FezHIygycH1v(babP$>-Hw>X*)Dxf+*8a+|uVj+|&RXRb3#~*^VJj}GcvXC= z*mq`I4YOV@Tb$S4K7~_H_PLnuAdMLN5=k_XGqbSaNd^AVzdwIcn2RJH1~WN7Gv(T&E~Y9KJD-|;tEb1SHLM0oa=sFog{P<{xEoXztVB+@o>g;d--rt zM6arFywF!Pf1^~fdJa5EPPV&Wy?#qX4`Y0>>>q_#l~Q=6wZ`F|`*?AVbGkhTjRxIi zo9eg*K71Ni3jd}=Uey+nvx!Gy$3QQJo8eiNF0kbCZJdmyvW|{8^f7+1fB=K}k*%yn zhZ^|#cqa80HT!Wa5? zFLu|<=+KT%E-y~^RuX%EHchL0@!v2D!~I}RaK|D&_~_rj81V*DbrMK9j?`kP)pR~M zD02wa|q+d;xR&uoeN<&F&lgE*%(l3Tv^+cY{$o8rDb=YifJQD^x8sY}<% z)S;Z4?vOtgeI9xqe^0%bUwFT6-lw`rDeU}Pq9Tg(D(8DDn-&U=uRc=?*}$bMvvsby z#J4_PuCyzOHjk#n+usab4J@Z#^sufv(;y2l!#q!8v%ml|9$#P|W6GK~agSjB_Y?a> zAHW*fKy*x`Y^$*rF36Pys|!d--gN<>7lcS5#%mT9R3I(2uc5;Vf*+;?on}>GK{I)B zQ@0~m{XwQ+Hz=1HWKS{?37K4%LLEw?3L6Xd>Ma4a^24<=F-+40#xgH%+))%SZ{ zb;vhBO$%Jy{zj7!Qgp|=zmj?;Z;u;hWQY%^;TX>rbW6a?f@#wB07mb^+0Zc2T!%x~ zk^R;h(d`Huk))Rvx5HyKjAy?~9P|ddv~Xcu_hDw}h7KaK_P$3FmjKxO%wBnZrKKW@*g({(E{KTB?aj zZ<|4s<*iIp<>1a zMHDW%SsR>P+t0C>Ix9;_oa1tBo|&;Et`HI|F3dgC|ElPGnT07-+)rRNn*xiOPBT(ze^ z$7`;wExhNi|Gk3GJIK4h@$SDYYx7m+?zTx%q(T5;YN`N&K(37W1oLcf7B|i$N3Fn@@o(T7Br;LK&P*z+nKAgQMwrwO(q-emawD?ua zS2q|CkEjAYZMEqj*8NWQeZ*0~!_Dnp$330URp+a#i^h%-MC6^J7Va|eL6Np{8*+((A;6;Z~RWc&~mUjvpJ%a+|> zFdKO%B9HZ0%jJxvidzuW1!oeGAIX{>!7P%`W8ZM=!ftJeFh<;SyYqxS3A@&2Jv!PlR0xWH!K((v(;JO!F3-?&(3%XMntF;7wpUznq3+r_ zZOPuX7AM&C_wQUh&;L@?Ie(m_M<|52M_(@ar5*;-pO@Dg5vHCgkWNt%;nu)`@XvC0uH;B2I&icUx0UBUzaRTv^FeQ8oJ!0|U z%K~db#*V!)_{FS!m*<|+{ZYpi?!&ReeSEf1ZN!fwH!RJnXv%t0B1A_kS;!55Cmn|$ z$>11^XH^zhw$Voo$6`FjlrsTM1(dROWg`X!XG^|aLF#eT83Sbx?|HCkj@gq>U z1<|?q<@NRO1@?yptkr~h7zu5)T+$hQeHS#PozwbagWI*u;r6H;D6 zlX*rYkO)xK%T2`6YmLq9VVpYp0Wkut%NdFoD+8d$03crr;q2thlHlQ8*O&3dX*t@h zLMX3m-*^s`TbR_>Js~U0KjCfq>;5lAD8k74K6Y>TJnXZ-y3pW89nX=!ayeMo15P4u z&xwb>xSZt4hty_$6>>DkQr5ll)n})an=N6q^_iCptuO?06#%oG9CkKRQ4CKO$)BuN zW^WBmYrMt??{_HmO{%CMW`(m}TT^cD>`*1Y&aj!rh9e*SfU`Of$&b5Q#EuBvFm`ZX zKYoYw{J`dLCGPN!HMKEJ@Z!x{=DQ-E`;`|!p917P7d)ry>tt!rR+Tn%^eLZAQDqe6 z&9sVuQ&SP}VT2tvUMs=?puy-HvwjM>c$=HfuI zpa_JJ(w*e!gOjiHX3bz-Uo7+9HhZfi?I;BZtNq;hsGw~rkLptFo}V0TxHVBcT0NWH zY3QiUhdh4wK2!g$U>;0#b7@D2lqd)e3P-^HV3AS3LpEN>Y&ncjI+zr z{Yi!K`FWO{n@~*KrCS2um_m{A*@gQgY*LIiEh#a|RC+C=J|y`-ZLY+= zMpK-qCvRI?$xWr<7ieHE#}X3fMVFc!=dQ}m zzha)ER~?o( zd}^iYPJr!=h-CYaxB@IT>89!8Wtmjf?ljT5dXvnPKj%{8$zde(KNN!(VU zN9VWrY%i`N$LynAw$y<&<%?>zK_|D)19zYvD)OQjoR?1k=2k9%vFARm<=_W#-$c+ZN;CXw^y?hB>?>LmMX;ZGjiMalGR zX=s%l4H0)Y7xLChe%DjL9pZY>Y0@RLypp^RrY3j9MUlDWZaJO9aRC$ zasFnLF?ak6cBH6tAehwyOLll_02~h(U~#TRX=|KZ&4EECU|)FwOQgAu03l$RAWHTr zAmBlfVKPdmNg#a$VN3t-7u%RDbz+Yvepn}9hkGLYT0veZ56BD!82A0XKd?a;i;p+L zPh6Oc>1#b|A6D=`c>_A^`)WQJpfMBlgIq2Lbh!Q;Z?KckmVoZiRkJ1!9NKqD%eT(vx047 z;cSi4{XTKfkk7if(j~^E3~$|}$n&l_l^JhoNW)%7?6*?$%w1rhKc~HCSXS1d)Pwuc z;iB#0EiYn{0R+BgM=BYDP-s_E-KA8~2wOa_u;GWqe4JM!)1>@aJ64qcVK`fyzZVxd zN|v=!dV$fL=LQ$kF<-~!3vzjLYVFT0E$>%<@XzZWMms64OIg%tHY7sATC8xM_@1mw z@PkV$Q8336#@XS2&(nTYP+7G*G;J~@>I5&Tc)6k)mnffZVSRxT7%!Wk)b^_cW!0I6 zyzhC)&l+3RIFgQGo+NddPMAOCtnSROLJeLXr#6y`vUc`0s2$tfvV?&G9!LFoULkst zq@We{NdF0}V^mM)9`%OPVRd_9xj@aeW^3B+%SPiVjLYCty$mVWzt>~;>#h9P`bYuy zjmxWHUiI8z*I9N1>&{lrUPx@2A5yuhe_%{Sah6ki65={tY6PTm-ER@L-CwYnhg&4_ z*QIBYtT!EB`e&zaK3BqF_PIP}dbsM~He^~+xUcN^yP%FIjQvs@QX;tY=bwuIwRZH_ zZRK*2FlBAK zT8%=Oe}TCZ$l#o>vwAaV3dCS=P@&rZ;swVqc_eb1uNicp`v*3ms6rqE9++E!p%UvN z@T1#8*kzr9JawiYXIQAX^xvDC;ce}ge>;N_#z3}nRe=rxH!xj)_y%}52=TU>h8|!> z(S5kkl@mE``rRt4s$%$RLkwu;kX=Y4YklhwCU7DrLl^N;8v8=EQn=KMektX%Kf z*!4)$iqrBt&*D+F+K(n4%RPEMaC_pl!^^5UyK1-R<;jfNdVCCzMQ88qQlXwZW?}qH zJ*n}g%Vp^GO5bbP@|YEr+|7fr`jBM5F+Scv;O9{)+Qu6lxZsyu9H^j$R z_WX|so^yE!bB~4&)#(tJ@)iub3N-V(1K^ib?ZlG+wHZ+*hkYN6CLe$IL#mJ zm=X{W)`n{E#-wG0I!xc7NKV=AG+a*|097%yAIN=PalG&MCd z84uPX&D1$<2)pKI!aEj{rF)sfXA2|OwBvW&byJ01t~`6rexr{FTP7#@W{IN@9hO$A zt#(Jssc5?8?K1aO_PWor63<;29n@%+z1)cK zsXv-b>-VoRTo?XzTV(;~4OMR5F-kUq%Dlf%H1?~k!?M~%Ta{mf)Spo)XJXk8T9+4M)0*qGY&j@M2O z5@=s_64mdyxNsb8``iq?&D=J(vO`hAm251{(cfL6tKa}k#ihq^7{;$Gw@#f~y%vnM(6K|BLv(V9p49DXM{Nz&x9&71Gmjx{% zGFlVS-P>=3wWOjJX=nbqMum}i2LFh9L&=y`*>L{g&a2Of$H#jHI&-u2*36XR&-*uS zZLKf$5V?4&$`J8&lxGa>DkS?fbN{(^1*T)_BZgfW4~>*jX-#(|M09CP({A`H4_ixD zuW5hGvrw*BsdwAfm%i!#1UR8%AN5=7ev+zk+Z>vGj}K8K=}vVwsW`fghAsC%Pt>w2gY{Q1FSLr{3$g#n^@wC&LGVk z5q1rAbvr6pSpkjD#l3!U3lQ{xG(`z=&BLp|!OOq|M3r0G+R7ak+PcjXUs>FE`>RD8 zmP)8Zu8F;};7q9^OXQ_!!?_%~`kZnGp;`df1#z)fE+CY^`}A`I(W-+G$U1pDn`LNV z;2UsC`vYzee(aP-HUrj58~a9|JdylAE&v?R5+QkK5o(}?fdBzppWj`77&JpVoUjqN zdGqEco%>!%($dm9+s(KYR-=Wbk2MWspP-?gwIfGef;FAGoi@szwEG9v4rt`?fj3}O zLV~QXuNXx*CeWtGO459!K~JO5bWBP{W_KkgQdL7^bfZpNL!%X`a)CyE*8#0dWy+3q z<;9B^LTP>a8F?%r@V&64)37UMR52;1pzvo2W6-LSh=iovW6ylXbB#0I^ABrw_D6bu zaUPQJv4E8uc>IdA55J8{y>AY0BfDHLr1M#a)4bw{X-j^$)hXVLy!1lVD{atHs&u^Q zq)tIl-P*tF_PasT@0PqiYYXq+vsfIJ=Z$2DU*uf`a zuMPaE_>VWsT{SX^$8St- ztm;~ij_YgxS~nHpygTGFg3LIZnYvUh)O@mp)}wFM^U6($?{M60;`yh%L~Ne5H(s&m zFTR`5`c97eew-v_f+Ja!KlMmL!kiZ;SIskkJ74 zk`jfZifeG#>>XON5xlC)^km9N*TX_5Gx+5OGbIU(Y^Yh}x z(@Im)f+A-8l){2PD`Gh6t4WR*9J|IGSRGY;n@jc5?3QlIrgL?VsE)b1l)2@&$xthM zbB)StL6}!A&v+f!_N9vK$C+yDtMs8;?+lb)T*AhLq&SKeO}}qZGWV7J>!*cv8@uk{ z+{N`}8(|W|kr83+{@_Qk{v#Pv9bQ{6+f?l-otSW13L}lbwNxwVRKF|f7UX*=MKZ~Q zDe?!W{Ghnw8|y8Wp40Y?>g@MT1V*iv+ygV4S!Ht%HfRKQ`b_8N2?sxP6?g5s5H%Am zq6B&IlN7zK(8;x!<9v6JU&?=-%SXG4eai4%lwXIT?1PU(ZbzDE78gx^+e1g+%kqwg zG@nuCxJ+5Yey{&fsHmt^M<@r2R;EIVUmao8ydx9+=8`Pf1od2 zTC*7cya8RtF|~JDzVR{QSsw*&imJcyu6gOw97CL7)ZQ-|2`AQ21TO^@b&eG zdee=7l2TDmj~WE4K7Zy%$!CMFSXY-qm<$7C|IpTK(~5T|Lh$28Wm+IBxBx7Q#`HK@3fyn-@$oy#l^#F7%@iYF9XTkeSxs%mQflQv9vSbv4g!FW(xn;iIIxw$CtNvIVS)Q*QRGzUBU zmX$i*VzZ2|nhm+ReNWvHnai8EZYA-Wb;VKJ2jgH^W2l;No|EY(jQSu86^6hIOCV|8rAZff^}z533-Pj+f;(+;u+m_u}iC*vY!OZwc zs0v|gkVmAG+g`Vg`~H*9{~~k#ZR_Xa&o1no$`+Fnw9e&Npep( z5q|4U5h%A^Be6^2D}6o^-k03*`{V+*@@``%y74SnZPmOhl8VlDDYE+IAKGG1{HPbg zmHw?;bqbP&-zJh6?x&#~gT6S)PSltS-+wYP{}QKcH-~s4w4H}ve6nB{+6tJYJa zv|f&%B7A;*jcq)1)$-+X#lCeCpWC@vg*=8Wb`HsD(0v_!^9}Y_xgRq8|M+_iMmA); z+f0(G(|`RsGH_wdW8q0DzMI=k=^RYM(HB9he1Wp(N3wONKNQY)Ey&U+rKh9o128#% z4c0Qv9bWHwCV_`1g?}55oY={x-s4@ z@ppHjs-rVUL>7*v-9vEY>r%X1-vaT0-12{%1cr;_QHxo!taFUf8!De#iTvrX~p#7>LIKsAd>fOuEkpGs`I{ zNyVG>fx0rV-|(gJ6S@&J2O%}VG*&s%%*j@O64b2C+astS?2o8DI9|!=NflP(F^(kT zxdOovdK8523)PL;uWxTqOibnOJH~8mY;unuFG7u#^XU_wJX*--m>;wi0_Qs>>OAbd341ix2u)L8{gvNVY0>Y;d zpm=wD59vTNddWikg#0Qdwjs1gbrYAFns|6XVIyw+EZ-T_m0@!pQF&&YBi;X3! z)4xk8U}L!06{nOY5(nLh9y$(WEWg9s4h{=jkH2ipPl0@%5H4mR+Clho1l1nsRRT4O z%%CTODh+B$cFkf#s3ReHe1i80Qmw^dE}*%LIEKe2BvhF9QkeH7HUS`_UGI4hTB14d zg;G5xte&dkcRtK3-jEY9>8)910CI)u`x{$!k$?;;$~*PUa#K|#w+br4hx^OI1&slkq|yI=J-Llyn^+H z!;jwdQ%;8PHkYKxZT=F{R3P=H};@`}|ynT~xl@%uDh^j)%W4!;94jE5iQS&#yjx z@x7EuVtZfGiS;@k?ha_53UNVmLh*o(T9PRjogOb2aQ>Z_gr1Chk?x!`pUkUKMJjPU zH&W-D)2p-)ag>2E*5FDnDnHkxN@C`E*3+QJ8McN8AJZ_dOo#7gvlv(YSP?Y5^f)}6 zO57DYrMD|8>sqR)SNv{;nX|6?=KBw2(N7Y5d#ZK_<>eo5>^nVn@aQ>d8pamUP|z#N zs5^g~n_Ccr_o8llPK4mybm!ey8SezFQLLIO(Pe!UvYXqirz}nwxYw^?;|tK67}t(= z)TKw6CGA^3tMK_brWm#lnIs5{{pjo%Qusuz1+Hg(#13JIu*bX*G4WzTM^#?lTj{Na z-j(k>(|oQN!5t&`-dh=j!cO_AORh<$Suvd*N{mCXH4;C()7vvLI z@SRCw=LvYBl6;QoyLlt3gy?gOxyc@nXg*VYt`^StJ*3k{lNtpNDe_g1hEN4hAgot6 zS@QFOE)T5o%IfG)VEWHl^8|L5D?z?51;s09U7&4A09{^4)^V97NWB3#J-6MIkW4s5 zD**aOqouyB5J4U*EB6VOjO1AF8{XI3?;Xo4dAWIdps}*;62Egy5Jtx1hp<6rFU}86 zcOqYzHV58D&K(@{>jlHXou-KW62vY(vo0d#^at`PD)=UCVRRxQ$%tta=x)6hKb^x3 zQCmbn7Em=(N`H_9VP5s=({F9zlt~Xfczurls!*|k^5pkbXEvF=kdT|cAUz(Fmevc} zF+|sYIQ5Gn{TAT)b*7syDs^B}|8iftXg7e5hm-xjZ4gR4z^Z{v*fCetk#&6?^lGkx zb{#g&gG3&aHv8>g5=gKEBHt?zFqek_2?8-j@Gk0({}e&Rh)f%@r9;WYyoIh^yCx+o zTLCp=VNp?MPfxBvqi<|%?7-AiPvC9Jj=nzgKO-MNqgX*v@rB){cY}FF5>$EuFK3is zqe3Jt0$x=`U0uq@M-&VQ&3<=8uQ-bU8SwCA_99|<6{0kz3K0((Hb@yk4G9@cG`>Ux zYzfTF%!r7IT~4Y|@<{QBSQ?%#-hDOa)_6j_HeQNIlK+Z-Zbv5|Kq3Lkl~(`?3WSb0 z42>p03PZRz;K&OS(4ci06cS<#Q4__?3`4duEe?pMwqN^}WJj z<`IirEA*(KVP7jO4-eKMnAxWrQ`DJHF@h;e&xINX? zMcyhb{A}>Z4bR!*5fitji+a5;DXA}yk;0=Dy|Mc3GR7+`?V*QqOatCD!gvn{EWM^` zb-7646M42i?wCV=(=e*Wd{;MF9v*h+;H0Fa3@auPOo{X)ob3v6oHh9KmU|3acsIB> zecf8Sjj5_OoG9SKv`5PE+eVwb)@?&-#FFem?(25nLlc4G=iTj!V!sEP{GrWcX&e4i z$Ts7uvm_U09p;3TPO)K){_ zv8R}RdE%cxg?6)%VIkQUuitQJxiiPuC4FxcKe+{=A!5*Se&&P*b&oUrVAkuMi}m+wf(v!D=p1q_=}cg%7HXThIDr*^zPGgx4Swe@mu1@ zWYH=rbq9UVZL}IAhWISDJsa)gk(ns`QXY$$jcK(&Y0fWI*qFkqH!#hlB=49PynG9vG6m*)o5`#v6@{smi3J+h21q z=mu@_C-tBT%#15#vq%3ljEbrM7uwj^#>HK0(>(T;cwP*l0B7w6eb}Sfi!%ejX~=}^ zX+g##A$AOXmo?bk1CF2v$drNiIN}NR`}c2W_74Esv}9bITEL#c*BMWC0E{4GqZt+5 zL2Sz7c(ounDCp^(v1ctd0(&4V5uKRW(S5hEXQtlk2pjkv*qNxbQ5Bxsu_|D#5Mo!Mh ztYswFn8-*7u;}0CCtI+#1?=D?{ucx^k;~;~G3Djuh|1dPZ~^XCchPS@I(q+35Iq8E7rI5<-!q&-wfdOW=j?*>N-)?WbITlqwMs zIB=NXK|luB7kX%DILZ0Uu_fQ(w65(to}YRkZ8UhQf(8jaBja1BGYRnV-=O57-ax0M z6k%AuU9|f86JRJH+2;kHjeJnk2JKxq>UlLa$?NMjAW@E}c0*JH4etY%P(o&9V0?rQ zuhz8$<55%hK6!PlrmCun*kDamSm1gcY?>YaT}4VxLx7_p5-ujKI*XvuXt<4ZL^pXX z2WYV|uR>Z{1kLbe7)E1=P-{St|6S29rY2VO@nb~e`F^RXxjC=Z2n#(uJwSh9RKlv& z%JF7mC+mE9M9(13USefsg$NDH6ik@dWZ$5K6wItxt)I4QH?QDv`J)cY%buNQ*8CLn z3k#F5N)3C{C_cWuu25Qr8xkW~FLnrdtu7@+jXxe34R|m3&r|cZ-g`&K3Ul-d3SVNy zIV#{0FZFqPQK)wudV)=m`M;u#;zLo<;`G`v>WKdNdCKETJAGYW-b~eIi zib-l~!(6;X8$=%;gaxnNwep>Qtn_&^^PN{M`TYU|15YD4x~W7R6Ymv08)B|~8OeGw z{nd}Xf4qB>C;3&~6o$k~nRx|DUaGw%8@_h+7^k!}hpeJMWA>PbvJnx-b!QG*LcYtc zs?rLsGWcpo&B(|I%~^Cz3?lfyv$LbnnOa<2Tx8WX1q}z%-pR|~XoU5V&hG9tSJxOb zZPtqYZ&z`nq6WyvkPZ;e+aW0UR6`5csLdQqi^sjX(;`jR!K;&oBR|AqF({cbfo9O3Mbn`9wU)uPhbTEvCFY7JXDQVPs86ivab6R;?>1jCYpD8_(~-ir>bL zP09FLHFQOY7BYA~AfsDI;EQ1sDLeedrGbj9E`QFsfOR1wC56Gm!}G9U=M3>&gM$H} z?ty0mDaIQxJxKRhB1Z!pzM44k;3LcO4Ox znD;6sTV^vi1MX|H9ha!axxGqFc34aIP z3i1=C99Z8OMNPTV`=KTRRIupzC5OQve0<9kik)^h-$1JEg_j>CDbps_Qfntq>{s}& z=%{=njlHoSB~Oq}YK0lx#rWmA1_m43Gk7|SbsNI`um7WyCyF@EqDO!Iwf3D9|hW_|L5$|)%!yps!9lwYIqjqRw;M>p> zvH8;@ksfnC$(w;?hI46QDcsSyxxd-jtUc?~v#vEG2fD3zvRJ1&mGMs96K!o)FV`*O z;r!b@DUI8OI5L9lmG|y(octy18y__;lj3%s;3k>zAeA<0i%w15eM`^ymutb#sJRnf z=0}(|a@~<6uO3QE3!fIWx8*@dXM~GO%V0{FFSK5p;{GIjMUT{-iPP(=6%VJU1zB@ zKCDUnD^xqGE0@h~8X-*;UH%L60>uM7aJ0#AQo zVWE)2JO(&-%}(gVOJ9Cw^6dPsfZ%4Mo~Y8X zs%gthYYE*%AQAt-vz`XA@$;QXa&g}WNKfDaPVq>M0HyC!DpFFyn{N?`K4_p2 zWGFh?A*meZ6eZlgjTt|F=m1bm)PBQRWGzg_BvSNx79ZZH{Cv!J&2RKeEug*myYVvt zCR;lO7yAat!Un5aepKv;*KBKXF$Wwk)qfLP2Ngh|hF%_W9oS{vuGf#$EYe60eAhm+ zPgAVP4u^yrjoR6)-70rk(}W(W9hqUp-mLv`lrw5Ye$R>sdY1)I<2E}z+yVU*?21Y6 z$@L)nK*__kkB(M0HV})91X4P|gaC03uc}HU)pvhWX+6ICr>KN|WHx_L9acW;1EMAwY{y?}#&P{q3k zTsAl8IB%>@9QEB}drh^CZ&|pZl(RDD#K6FCcde?Hd?4tGNO;g)G8C%Y{VYrIZ((k3 zuC=EC+wATC4UW0Vq;TGfR4_^5&)r$MR{HUy@pSU2knw{1_DLkK)0j>9@!uw!FS^4^ zg@cxIiv6^MNm6r(Wi~%+-X6?&gr?nh{RH?e(=6|P{=udJCjb~wLX=y#yF$rtKQ20J zGO|LCR#W9lshL8sIOI@t_AWrW)2cQmR}?=Ff`id zZ`UIt31=K!tut?rp0qJP42u4}xX8EmMHmjAiP+I0gJy|tQ^0&DYAUz#Ni+0;njf_S zWr#d}UytPT+Vn0zS%T&IyP{YDri+&bCO_-0BWb`__e>JeErYg&Yq- zUdTs5$Fq$wkG`SX`LmY@wKu*d(A%=D9;(BzW)jC19YEV9mzBweO&oXkjb;(!{rf3l zVc1N{sU5+D%oD)uL1=9^ZrpgL;o7?BxYNm;0p>gk#>VuBx(K4~&&!KOcGK}0Wbkgj z1^CY00=I^%%|il@n<89cNx{LvL68}w!m_fmB!n+O^c*~ZR##Rsb8;jdT7Zty`kDd@ zgDA(5h3F9;hN-p;<`wg+VkQL=6B(%wS=bbUQ#Y+X<5D)7X>T%JRUwKVCWyXA|NWhH z*71W!v{-(xZFuM+SFSrT(MwI@as4$~Jw6dVo1PI<#KXX-Dzt94`2Ccn10$AG2{}=4LJeVbYYFB|Q6JWp z18*OaW*jw3h-enY0Z$4sDbGudrNsgU6wIC;f@{uhUGu>&V8}o+hQz}}M<>zAn|RNf z1myQlA#>)|)m3bnxIZ9`yz#Wl3YL?1pxu~-n6?cb@@P1+pxjt9NnC^Z!o@}5@XQyc zO9*lE30&-Vs#qkT={?fxk_xvXrec%+u4suYBpj`$qKpKLf zW8L9Va<6&o90ZuyvLkOX94evLz2+tzQFNEcurUFO^=~qSIC?cUJ|28F@=cxp{_Cmn zU24s^6CbWwlviJ$23nHH%qH;fQ;4YV@!^h*9RvvG2^ZXM(d zuX06GlJgqsXvaF@r>+P-UrhCEt|7VoKo_`C50j_zryH##LGDnw9hcDY}Zcvvv(^0TN$ z|L1;%=Z|ry*G4=rB*l(=#Jev$yRv&l3$IgzC+P)V32ItkKC+|pe^b2$;`)=OE z^oL^BR{VBkQ4#j{e*HzJ@vj>A@24zBMpJXny*abr)A6t-b-Z7BHDQq<(b{%SM4geGsSCfWkq5dl> z2@h<;Og~+{XouHtX%XVuqQVd8Vbut<24P@bM`qb%qxU7D(^#2B;baMuOIWzGDC z zsGoRxV9^nw24zXU`=nE%K;-M>#0RD}A#+?*dUyIzQD#SIM-@Qlp=f0EoLbvw6Hh?h zDAD{`f>IN@DocXleYKocfS|WYP43?9AWgo8RP`&JJB9=F&d!G1WZ)At9hL#lj652i zscGy;I4uup2YNFnhpWCbmLSja!p#hPMMXvEsTlb9Vpdixq0a>OqM#sH9J&t6t^Q4~ z+ZR+e`?n4NnU6zT4h{~sw6-FBthF`x(13J?GF&laP@ZEj8U~7b0Q@;@1gbyqXa?w$ zoYxfVzq1PS3vlWJ5N;k}%sDbb+F<13QWho*Se6EBnCD+?R1})Y!)0J%0Y40<*hU~_ z*UP9~qAJ)+YX=)a`o7eZaP591}XH;jumX87}Sk={U?teT_Fj5yRx!k zhx6?r?d$Hv?5wQI6_dq6^iVd#iGWdWo~h0=kFIhhV4?vmh?vI%Atove zl!}f{q3#-F7zePigOV6#4$wp(_=a+tYN|2UV~iRu<(0N2k_f7UsF{loq=d&EZ+ub{HvlcklYbL;Ng5rWF>7z{B z70h~?x$kqiSq*$$B?XR`=YpywBm<)oGM;b~)vYShx3;$K{c4WX*9adTxaJXH#;Bh! zU&9j8B1ARi(q9j@){edR@?IMIHC@`E!G+q&H8N#rnAyIv68%f5jlf7l?LE-DkT1_s zwb0N|I8fEblcukelPSOnhJ}r79?DiG;3KA{Qo;Rv@?Zx8nHwG*9d*q%YF)oU8F_as zMOr2I)y4UPv%kY#E{NY$c{v)$1M9A}^F8pfDg1T@7so|M*65#ea(kcnn2If|Ds`Mi z(y}vLZwuXuR72<5LCm<}EW-dK&g8vEOBF#x=)nQe1%~yJmE)7S{u-hCQt(X{5!;oX z^$qRTUKAk9Cz~_)$#M{2B>F_?zX7WIkGu8!LrKXZpz~tD1A%7_Mj7y|14#zvvC`0n zLsft2011$76`jV`^J`Gfzv2k&%%6Ny_pGSc`D(~#f zua>U_!-;sH>L3D`Fp7`3zxfT2Qt)&O4u%@@4XmxJ!Gzesfd`b29@0Jpas>U!cAc%Y zHN+7~*0MP02~&lg+(B&lDC6Sj4hh#fM|ZtNKMe$9D6WK)l9H5?`N7RZa~mEoRTY&c zxCbw6Z9#*6r-kqd(z3v{9yQ-N{|3{d_JhU?AHWvDW4?XSu?)Ndfo)JI4@DKS7K%B7NoTMxq=GGDw12Wb#!27KXIXa8eQafWXYBCQ&@BUwcQfTkohDNHYYbRn&cFQW}0de-%qpN?fXs zyU7_E{?Pu9f7DaJ4FOOO z9TNl7!{4wdw}8wVhw~o;XcYu2?|iS$y{T|yBM*v3tynoYav_O=Q*W7BHwiqO(13k2 zF53nL;^>r=P?nI81w}Lp0PHTX*^_}r2;qYOcUG%J6H*?661;3UVgkDqv!Pshr3|q& z2*PmMG5y~FO9qlPWGl1fzBDL!(5b0mKnR1f5t^i@Dk^W943VM2gh?c@PZ2x?8hjW) z0#UFDt`X2t6L>{L#1N4tYkc*pHH*|7suG*ohBWwp0|#G7d8+E_Eyw$7aLrFWg^$mC zA$K>!34UKxv~c&;(Jdg(l#YkeVj=FsaBX@H4UbaySE3sHcRk9`!U^9Y%Cb+)aSJ7Ol;dg*E2=w0!-(%LsvyGSJ!j337aM;u$ z2ylp!S3TFz)P!!U4dMq+N^gA^yDkBT&31x~rrZ+=r#cdlqmGxtI_BvR}=p}vC zZXLnzMo0>p8l!IkvZTVhC{zu zsYE_y6c#G@?oZuy@R|zCcpBDguI3ftEs0DVYF{>XMr`~9+7~gVfApQR-x(c$V!{V4 z)pNQUcO_K2x1)1lXoiO{MtiPKhzP|{J!ge16|3K zA$N9TS2YUjih0G`-=DF2@HGp&y0@$JK39JEwdc80HH|9B&#AXb5+jb|($z{ZS{U9x zY}ouU!B2+mT9n{jZ!a3L+6_6xt2cT5Sn1=4<-Wr0N3~=^V zZ5@^=?CGro2COSk(jn_oP+=OIn82nBk{m8y#ry>wLeLKUfi{F5@P917hZpB(N0AqY zxX{@Gs|o601fkEA@Q?MGwh_n~v?LR-A%ntOp}MlFO2|gwdFe@ma=}8=^*az}*psOL zBfy3>&(ZSo?h;U=#KkjU>-{!RI%RU`pgKi3pyc3elmR>g+HPJzRp2+ksVETg1u<8s zlGS)4pasFlxcsiUX(3WP;^&5@^PQs6Lb4&KU_E|E%hv20UhaTUkf*1Ea!8I?tzFv>Kv3c(4n3@p#n)t<@ zIJEdMxxDh`61-dF!XVd5Nd*wlLIUlC3-~BRy5Z_&aWV0kAF(z^Xup0g@u-Z!5?8DTmlbdmb}JGFgWu7s5a@o8<$uHZtb)()J0#XT`A%ICia@=DNX#3 zT?L2y?8gbbcI!&iC+lJqHqtr|$}qL2)NosV7AsQE)VWeg%Mn zOfMKSnW$9aesE5?4@I+j135&tisha>(=9t>vfqHNf3V*#K8Fme?s?%%PWm_q8T075 z^AviEwlJd{FZ^-UoHnV<%NMi)yMd@|xRB@5-PwN+eB6M*73MMs`}aReAcGk0M3o8x+;!GZs6t&eAGh zJMFEe{M<_RH7FXzrzDLWG|etN<#-&v?_N(`HEnAZBN|s^kU*M>Q_kVEIVB8CF<(7% ziuC(#I4i|`20A*P+6!l2p}F)rU`iI`hUqC2&|&~$)i=|7_N*OZsh3MElz)JFdxoh8 zC>IPtmH{rsh6@$GCZ&(#Ns(txNY__{h=`0yVIm98bat3V5_m|FCnlCoCm9HJNbHX@ zGzvI%k6yfBMKEPiQ9ZqOgR6QBUCm~1Jb!pK_cN!S;kOxb15z@y;iX)#hn{^>_r74h z^%Zj%Jjen9U&b>G!azeix31H)CYwPb-VEyPq3hYqa9)Rm#FTV&eh_zGd#iZMg)?}R z7@l7t8LozRAALm4wDYsGAHCSFhZ+kp&?p+qLlObC+f)+}jL6||LAf~1rWkM%sT*M* zukyrcaek=ZwYGk|+NnCjrT(J3lXG3Fx2x-q%Vywy4dg-xz~K?h-S(CQ!y|8giW2fN zQ915xr@reyld)W?fUANb@y)z~wJ#}!IGiPW4ZTZ()p_WjW!KNi!&BB?9>g|lt9~m! zq)YIF-16eyp26vb76*Q@)<4$$<^H`f%F+s()pl$e)oA~UP~R(#?ppK8?5j(Y!on!VJQ##3I;-=e zH96azIC<0gwp{Uk9Ww9ne6O3cJ32b1E`;%$Yxa$W-}P5 zcc@K92!Y!t-T{wxiF3-lB=Mw*HSY)7Fu2?s<-a$wprmlt7we^V^ryoStFh7P986~` z;1O7JPEGRyll$Mmdj;fwy@qBvybs2BL7}10t>?A7C2@2E&2i^wIm19zpE<14l%A8b zdFSpgX%(3;qPNvCR@d&IgO!P=tDkU;@BLxSDo*`X0q?Nd2v* zywT7Ez~oTac?Az9Z@?;2gB}|mGq)KJ3&EW7E$+~Fze(1 z9DusHlDnx)VDNNPfD|$c^v}S}|KJ(dMd~C={05!~MHtk7iif;k^=%TfQpH5`)l#)4J+PWxCq4k;m_Bvi7tA6@;3Z& z&fM54Ur=b|a!%`=a^s_dk+Bh&!m`5XvK5Ux%giN~UR8yG6sZnE_DqcUcQRg2(**b0 zt;L>QtA0*TAH|=1>xyO`c3QM5pG7Ynmq2>!9WLYj!Z=vb=^uoi&+|8yNLqS8ixeQv zK@ky6_>_u{jyEGFCT{V5VILG0NTJT@3~;Y!W&TcT;4J1T@!}I#NOGyJA2SiUR!F{O zZR`A{p|gl+uskb_&yAl__p^=HY~NK%n6b(?z}1~Mtyf+??43se(dAiK4R(Uqs%;;v zCfN0&?=tD)KF=BaYkBF`c=j_Dl_OV#l#G;XC0}R2G}xFo2_X~1s-qy#{4R^+v;YXM z0?Lu%yC4U`2jp#tg317ARao?ImMJ%O2})L>7Q=tMQjIt3>h4A;G5jZiQ&_#qS+_ajT-VNX-K@+-TN`F zpmW7n9AW@ArNDa4dgW(eKIZjk!sZPWSmV=VuwM!SV5|2k=4G;h3JD7)f0OaaxfWnY zBvzlnn^C@|SOR(!M<*vQ&dJM$+{q*9t`Jpt;T0e3b}%4K4=l&SB-d@q;X0IAT_-S3 z$~)QvtQp+VVY$|SYHEr-e(R{FF9X`Dn{V-?Ri*{)>|y^UQNWf0z$#MDoh10s#}~Dr zvqiQ5p-56_jJ*pBF6Ju(S?={Slw(aUmDcxwo*mk{J39730=G{nxB1GWhcXSn2ygI@ z(qH$UPDRaLL#aHREt%59GI+|_SnNoQgoLi}z)1oL8=D6d^8RHB$;smkP)HXSi{<9! zBh0$gqL1bWphP`Yod%jckYK}LA^?~tENMOH-K4ShaHCk24*00DIlbQLm%MJz<57d6Nb&-{WIGqx++u9q8Xg3vgjVw@dT1+5`_t1RJibk-QG4XBv&H4ae2% zx6P*xgN0(y0G7}(-YXeTZZxV9mY1qz?))n8Z42V62{#3v`caXt)|OF$El|#&!E5A7 zLlF}b!>A~Gd<|*YwtqFjCkF0m*Kw$-SW{8Z>n0{9cK^8yAlB*WDLaKE*Fi-JofA7|w?Pc_sV77qfZ?Fv49dMaGBPn1NXKMZvv@Q6O^Y>2mL(_vkKR*5WJhSBQ0 zR=4lSv+rj8kDZj>CA+hi4cK#rjW3f!y<)wI`6+sd`_&B0y~ls5bgmD7 z-P5yY;1VezDbfrbwPLJ&X#m6p8WEAYk&)CKsNlM~LN3~8QrbH;;*Ku1YDXIWHI%My zI*Y8X-w)9Er|}Sj=&U+3;`k)1dF6fBU8-1*tV3#DOsS_M4j5Bpa_SA2CAbM_O_{xF zhNoARx1PIfXt*7nsiY~0tQMk7pY)yFk$ZMf;+5{zSKTk%?~#ss^~?wEhM$Q9{T$w$ zed1FgDIp>zDjfXCb{=f%X6|-D9kt7shPiq0Lzfp?xx>`0zT`(1SX!@mcJ+M_-MDIo z{&(@*-X_tUEz>x8!Do!08AGe*YMW^Ifo1-Mm;Kp=qpP{3mzGDUe)K+sXA|>IhI>Ej$HpT8bk>N zpW*0Fq=)R1Z*|WKziS&FwBw?FX&|h+`Z+eTAe4s!F1)#Ls~qnzH6_Z7pnwNV=O;5=-87q;)-vOAkbJ))NZB~9ty3j!hBJ8 z8f#?w@sq8ieg*CqtB}L z7~GuB!Ua`yb^TEC?<~y27{sw zXf^@8s^zk9(;FictP%k)UBcB=yA4!5?5Xrg-Nx31b=}TxgWI;w^XG>PTd%tF!zo|9 z{3+>nAeJ4CwKiUY>;~2yk0%^0zdCxh)N4S?FOQS;zKW<|_${ckPtU}h`zR-b{;2sN zlxlbEb$}l#11W@_BJ`=IzO=6qzCenev+sW&8-@kreC+2>%}oM$uD??IGc&(jS$xT+ zASc;f>GvXA_R2~bcVR1LZebi;jMDIf0{^R;(vRX8=U*E=dh|rYFuMI`suI(ni^_L( zZhU+cKD?7Rzl5F=T`TB*JG^K-b9${O>3*cWP>*$K!}pB%#*Xfb14bEHSul*NtJ}R8 znjP=|N^^=slW~x#M35y9=9>=H%C=*vqvi$Kg_x1;-3=dct}ryW(BuC&Sa0KhtkBIC zB`p*DkXGpGfD83w!GU|!+A1v{WZyiD!XM@ze&-Ot75vbiMnxse)G0GUN=<^`n-bH2 z%!!I<^>Q4mjaA{)m@h%{8}mdhVjj|)ybOM*|32>2hj8~7cI=0HzkYJ>ROPz(Mtu5j za(4T8R$p?QU!d5ad%WSef6BYtHgTaK36&-9sFU$bSzpZcoqxj&&9P?71>uY4`*J91JYTN4xS3l(9xqRCHmYT^pAak6VjejY!T@r|N!n1dZ75;k@o`vfq z3WaxJ?;Y0Sq%LW5U;e+_kAT8G@9Y}=S9|$9{zh+=pSE5wCMm&UlmLCS0pA0Dtk)c! zt0M+YtJ?g+dUjpxMLX_wmO2El>>BKouU%;(2cdruDJ}Ckf%UPwF1N$Kvs`)OS);o8 ze_Q}s2C#2{S*a1396DfpP4uO$L-}7VoIXr(%2QnpQJ&|oPWWQ_G+!BLjK=Kt^d+8W zcFC!+soZyEV8m7`3j1JfX(E}Zb49*#-dUtS^w+m63G|^m(~Vr5&c&srkV}QXw&~zh z2>lFdH-_UsUq%*EXk4a+;}D=ur7Sz9HaTaH#h`d5QPdmQYkrj}gWE-qnc#+GU`;w~ zLE8(d@Cq@o?tB&sc7ISISx_iM)fX1?1p-O7inCs~`|Xt8h*9(ILl5306iujk@e{W2 zHEvTUuAD(E9o{4lbeUE~NDdfH2vI3yywAX{ZGO{g-1{R!i~pvnCk?N10(*|6MKaTl zRv0vRdDQXp`z$z{gk_c}lQcDX^ry)U?Vr(dEy3;cE?!{0jq3D7Yrj>m6?n&E#+YK3zeELaU zprgy{*-UOv6vOozVqqz^WuM>z9Ro~R9J|b7lUM%?M7p8?J${#Y%Zs_bBJ2z(p zBdC7c#{?S5T6T)!c&I^_G*BSZ&o@9S< zX2BeWk;y^jll1tojh?$KxnS!OmJ&E9m$>mXR5!${2;9|f*l%mY0Fmk2|W$Gu(U>!=u0}^SADtt&D_{EBKIwS4x@Rt7x#~n zzS_&THBd*kPBbt&-6xXHDzAxW-B;uL;^b@^#F-*!j{QqmkBcHD8F;q+D~VsfRt`-L z#_!w@QBZhcnJPx}G%c5UhwQn)OIjfkIX%Lvqc+MH=`&KiNiF=3^*vV^p*4;`$;X(Z zv2p%ZIGvE7$$w<&aB%3@j_x2-*i_n32>p!}ZrY(yPx zZ#mS@Ny4s1k*}(+66f)ee8psKDV$vV;P{lkH!=99)A19LO3}WRo zj-U0Z_}l}Z>81BZWI%2KAYE|E{wi$BwfpolKg#q3K{I4!<-xxR6Wv#5`^+5o z&B~X5fRr;b8wRNYATHU=)FC8a9!qx92AX@r=Jbec^*8Tf-L`KNU%1XCAva)w(M#+}FcgV`j z-dpyFL?oHX&fbJ%rN~N1M)r8GZ_n@lI*#W!(j)i!jO#kDb1;aCzYd&|mQLc4chRC$_F|$ta<~~oD1~07`7=*`UV14rvGoMHM$gd*PW+MOSUX^d>&?8l4US8hp zZ-kPhjU9=#G?zvdL-N}KW7-7u)IZfa#O3tqrb++{z3Y{wUNv!zY*1o;TTJmer?&kU z3DZog2AX6`d(d;4X6kLVaqpFFKG3>)gej-zm_hwmfho)9M@vj#_~kea?sW?HhOc@j z&mpW{USW)}b|2upNSySrz@A0QE%RjSB)F==s%tu|sx5Q2;EW)}wW5Bx<0Y^9el>qx zIKM_`rtne!BXh=UdNbXd$O``rhi4k;{2w9xrZDgPmd^#dAaID-)I1KbpG{g%&TNltv0&> zgKh)r%)W*@}vKCmwL-4u4=@TrDlNY6_}prFr_4GT7WG?*iLT zrY-puO;iDOs(^#W;Zj8BR5(-}TmsKc5`~#f29k`>rygTc=t*T-5O|B}4 zE2{}#L4R3OyCH)fHNeKg!6p1-iY$9JJxq;(_AZ~+hR2P5p6($$&Pc0%T@GC4CpeBa zE@m$H(`Fr3TJZ~wDtH~~UQTzDm{Yis7N#u&_zaLz_hVZ~AZzo?Iw_q=-#Ptft}5z8 zlsgFrr6=!lQmG?o&i?l9L$&XzW4|WeAgmeVl2-Z+s)V@G zQiKjDZu(w?C}Rw~2~0qHOmZ&`P*4Ap+3&7ku$q@Z-#0Zy(F*pbz^;TCdG2N-c98ED z7Gh{=Wx&&s$lLOx(G*Dx&jRzBf`TB(y25!6jA0n9?|k#w+J0kvhR_wDk5ilbSTUPm z*RQT$$Iye17mgklNK4%o)AN9I`73$F>L1rf+rj;go5l)xZ}dEZ4&t$su!lv8KHqs@ z$j7)oz&6|6QmUWc>a6sox3Y|d()a~bPsv$5Rmu_xe$IMTbHSNxKAMJE%Wty`ZHAZk zxvkF&%JgY>h?vfPj6K}B8qI56rZ`3<$W=U)-u2m*k5}++YW0;EYBlocRHKwT_VRrU z6-<4+jEt6~)Hkc8hN^Bl)y|{~m@@%JJ2LaG3cLdJYAp1=_`4UTIa|7OS}s4I7OO0K zuSpuJ_k^4}|9&r}vLo8z6IZ>ejUOtm3|Pk=SA2S-WZXAN+1TDxzrT~kxj}$c9sh}7 z4V%Ep$RCZ|z*ld@L(!r(7^U?2evbTRN*Kohy=8OQ+=9us_O5Wu==MZzNuQnKQvkl1 z5*1&pdv}K_tGlosVTS7EDZDp{FD+Hc5ePmGv(-D%dXEW~V^(cRSF7VjZw=pJe6?ha zlSKb&WrdH%aGZjMl7T4xt=5Uu*fOzsqcEr@Zr_jNt(^c^K9VRTu(tGGoaU-aCGV9TA~8kB_h- zTLcD!dFM3r7|l(nbffL!*k8&fS)e9%TFQCE#E^6rpvsy8`Q=0RoofUA{U5*}4M2^x z9j$<)0b{R~4C~kB)9hMvW46eaS489<2wk+a02_BvDd_lc zk_DE(<}5} zG+}FLB);@!Q$)APHyxA>foBg|xJ#1V8bKd9zwzXdbUEUN ztHQ=%534_u5^?IHZ(m(N_lf zmd`W{^yBhx;O!qIDR!={Pb&zK7S=Q+H3bzUYv<`Xub84Pm-%fg`8W77Yus)OD;Y*J zI(!$d=l9q8@YIp~;e2n?A%8NO$J>DLw2Q+hv=@J8`Tc*N3}y(@e8-_VrC5@t|1_{c>$jij9{$KPmjRiWj8+H^s-XY zVJEYMw7UAQ3L^-)e53sq_9i!MHCQD`G#XI6ax^mbq*-9{@6=+wASG|Faonz);g?z~wPE zI@$)jQ{8GSd|=^RgA!|Tv+ny!By;Wy@Ml59qit=7Z;%%}D&!Yh_ouKLc+R;4pAnGr ze`-5W-04bT6rp;QHJabm+Bd@riVD;2!}mZNz0^AKHnGHP=TF zkBN-|w5f1RO3YP3GsfHh>H}63W2%uN5Lr3fQpn zaWQe?Fl`>ssXB8=J+-{!5w1b8E;jXy>g!>v*#6Sh*E**+4!-)~3tK%4`a{;TlT4>! zq%}N8K9S&Z#L*{ohN<~k$K-zbJAPsgykoAslI>UiBbgn0N4qS}IZeU?)RW~&s3SkR z2Lh^>NpOy+br0_)AtohGby(A8U5ndMJS>FRRm1t-&Ji~2OltiDmKUN-LLs!&BIWkF zI|CKZ?}r``q3Y*@t{%d5H;fp!w;T;U2CM0J&S2K$(sf-2m{;TaL{FbS@eZbhGp08xK`IYVDjAqH ze_#k*?Q`H%-n_5>yXC|QY@t9gAPfgGEEibtl4oo85h@cr=vqt!VsZjbS3581&LRL9 z^52d&{_`XH_73of6X1K=r%Ob%Fx>_V7sNCHA|Zfhh%AcPO)$y_Ed)qV%z@=aN=oVs z%w-a63`F#ulan);E1qS+i*&$!kd7ki)Ud{c(Br{B46aRtSP7~Jz;!^G0feqF#Q3c7 zcSAG?_MV*n$qdcUzX^U%ANFVLnZ-X&KuZFTlrHM|+lA&6IEeqdIf3ga4jc?-#Ge$@ z>Zd;yB+dT*n(#en{Qe3?GVrD$&pRqQx*mQDT+Y60%ErhKAYNk7I_)mfU8K)7dhRxF z`CWta2{}-}vHUIwX<(+Pw*>_-h%%n8=6E7@U}=UP(7DoJJovRTuBjKnpd=J^ zLvPBUzeQ_gJ1zF|&kU#Wh|!j1Re`@tu1Ah_&Wl)&7ZKNq{j7H;-cNt`3ENs`+H7+N zi|4>>=F^u61@2U_6;26__|eJt$vKXGow3!%=^Ck)MXFidp#!Nf9Y*){Z>0!952WAS zcuYZAOEtm>m-1mddCSMT)-y-s-GlUl`mL6e!=1F&L2!_R`O8|B2r%WLia`A1KwA%n zxP^_4S)i4G#lXtR`Kq##sDm41JU|2fG&P0T6~G2U{7~$_aKML&^rQ`%BmO?`9@WJm z_l`1GeF!*QXJ%#9!)64932-7@?P%uTIl-3;LEaB}1M<}I9aai)8`3Z-6WU)VJ zYjbnCq&T;rrX~f63N0;VD{uJ4hHP^nXa`h)2*a$B0y!3L2s$lg+Lt3qR1O)L{`*GD3?w~e58oY?z zH)kSQ4YjG~P}055>NmyY62a8hX1Rb7bQYmP{BQK6Ry$tA(pyCsCHf{WpJQ?R_;rc> z$2C`;Pm{{hraWrJk0@4s;pq2|Z*6YI8WIWbXS>hVA@k6ARa0G~Y*+H}J9C>q-fZI2 z0@aKtJvl8YNmfBh>le$a-@auQu9OmsmAlj#Z>wt zCTFUtgzro3b0D;ZE4KOGV~eiHx!ltZy?<3ffpP=IGh{`R+GN57OGHI;S6zcKS5r<} z+X5Bm|62e}L)nbUoYH*_B_~%klUR5l+1D>H>0+A+E$>d`eczz2X!70Nyo>BKBk#z6 zl(L!H=x|!^2>AQ^4L2Xx>L2wgNVJ1(BI2z8C*M$(AW3wWB8vAh!$4I`ms#7IThV}9 zC@{m`>W7Z3zJx{=8Av$zuJGk8z*_BG-L(|Mt8L|z@A|@Ch z-UiV;6w`4q)Ph=olA5{@mK)-jk(fw?m~OyE8$YH7iEQ0$ysox(5(-?1>LIWA>3ks8 zKLY`F_LtSw$=|*i0r=Feq#Q5JcEj4i0WWRf@3|pe2jZFNFjEr=xhA}4hOk$JpR9m% zK^Me`bl{VNco%}l08nD~s?Yj$TG}Pg#wB05DBzJjkmC6>)7lx9H?O<%lEa67OA%$FRxvB%v^}m}QQyJs9hj#3lTx9dv$X7u zQ#N;3i~>E*?Z?Fu=Yt`OpoO{FpAoYedFx$PvloWE0E+2a#+5GFB<|5%}ImvjcCu z9=W@8wR?a4a?7to$*e4#bIY;Vhb3hEal|ki%;#G@HN}P_Y~_aDU?~;6cyXz&=ZT6m z@g+_nl`{8R-YDn8S!MOg)Gy}jZzQ=?iKndsjJ;8p%+1Yh1Gj-hwmrAypSE7!7>D@T zA)Q-so_{8g%bULSDU<%bjSqKi)a+(AWL^&L4e#$u*aQnFDk8{ZQr}Vt9vDod^3Y!@ z!9Ro50KzHI#Dc~k7%6y=wT_4sPDd_IuR*!K35fm6BYCPI1R#^r}1 z%#h$=y7jFYF_r+0$B>n>Jt@_=u99?)X#q;n!6O z1rYekpLO>6EvDEU)!2@E{jOjA0Oi;eI2r-H2ST0(&s4`hxu1Oy_~UM94G++{xZNSa z4#uir<^se~q(Ke{klgn2m5~};a8~(w89a@_2$~nFE|^Bar`J;``QgITymG1}4;xNE z2d{EW4cK0Hb**+8{Cr?(!Rk(-B&b5%5NKP}tp-+qptaR}La0Jr-43hi z-=f;eJJVwf+Qm%%^pmB{i@0m8`_(;?Hz?~6BNHUa$kmnL1h9vG$UpQ!IIQHYhjLskhh|2QP<`+!v{G^a@kE4E z0PdKvJUIrUu&^)|PEPT%5KZ2UrC7mlCZN8CqOq`=TjFGTq@a||`0SU?bfvk%7gs~D z6U!QsN5sW8DS4+$%t-ReJKtq?$Z3g(9)>_pJToJ4gtkN#1^(tp=;+jy=Vl>i=Eq0wSO8L zb5&VX9ItUv?HGTdqnmwA*ywXUPm-L6iku8DK1U^wL6=ssq>xD_VQk0m=8DG4{4}Na zUfAM#@mZ#4q&LJRMwl|0D{6~YO(gO>u&?V~oN+5TC!NH@8<$NjbW2SEoRO*g+5fFA z=Ib5Yil`q2`T$fW%akDKvqrw1=1)MGj zc^2UNbm1pVRUGicW8Pd1<;B%pD&w+~e>-^P_Tl(IGFm}=+H|@zfQOH7n#lyK1*jPo7Ib6G7&^$QWe=qlh!X4nv|k0ez_z};UMzT(IH2wI7B)JRUBAY7HxiLm9&{jq{%no zj+n4}m7l-x`5Iw3en}bjKswH1ONlK7MuWcXbo2UNxsnRWari>o?e+litE~Z;=LcF| zzvBNKKKV5B^zbI_xa?ItqHp_TffEfXv zHGFa{EiJn;txC>&|5xi_z^I>qo05jI;9YUnX*+|f>&}DkJxX#gGw9fCH&w@BBtc!% zPhM+~=O9yXeVXuj{3M1EC^@@kJjaSZk1^(wbveSUg1Qz(8}$EFJwMIV`+D3rQn9jm zGFAB|x-vqkTt`IOT-N;cyF6@Ok75%0<0@Q%ERr|WrY7{y^2v*TIneoInsvZ2#3yAm z=i4tPUPeBr2^V?Xg<*kyoG83)VMDmT4c)s4)@St^6~R`LaEP(o9OB8 zjwp4Zj|Yh$jNJGD-vY-)L^2+9l`H+hc}Z>Ub2wb#Vg>gHWLoUA??A?>FYDPKoEe{I2UJ| z$@*P-99kiYu*CjC+PV=ibA|kD#1ueV`^rbq)zrhjIyxE&7=|ayJoHmI9U2$tJD|V6 zJIS0I713n|G*Fds3(!;ANM z)Dd_L=0T87g%3^-Tn#@N{tYw(PF>)sUM1&Rf&>I0ggXPsR%P081!k21-XX_50L1XF zL(hVI11d$hM!-k|whcofc9E@GuL{y4T2)t?v;~cGdojLdRnTih6Lxy>$buU`W}e%{ z^Jj&R_^XhYe}1h0Ic_8rW!V*>3Qsio^-a=`eTk0c(VI|tY)Ai`rFjXJGpULPbT?$K z5iDj_Zd42kG?vC#C&d^`%qoEyt)Ofz_N(gsy(s;1W}$Y|N+>p{L&UUoP13x!4vH_s&!Fai<4u z=(GFV6VUL`U!o<2$*TWNG1sMkLz;5mx^0k3n~m8^-H{19v>-hxX2MQ1;V(lUOBAM_ zKt#wb%KPG(+v*;V?QgEUvsMeGyYuN*LkU84=oQR!U2&v2I`c<-ci_xI#F;?#0qvWD z0v4Q-oOEYD6)I;pe>M!mosQs+NO868l^yT9bPIxNIT7*#Vh3aac@%Km@It!{2n+=k zRl8H&69~RU%(!+S;LsUh%ZJ9sBm0}P>RQ*qK>!@}z?=mWrgP8%(9Uo7(5-?QI*dSa zO(B5lEBtTReyUCyN?^zl2&fDavH&s$=+vP)0F3geTd)ObsF|9Iv9Pd?pqZ9+cfSKn zA@CYPN&`^127`y+O4j2QL`;NG07Vr`p4q|?045|ap{0g~9njLXU%v`KpL3P#!!snx4zO6FXJF7y z^XINb2DB)^#+q~m5QA>?JJ3OJL8fax*WeCg6@Z)pPXm?|pc;|#FA*9)qO1kFU4AJW z%x?q)kmdUU(&$*(**n0VD)hE5no<%1Mh6E6gHI;CG`}tEZDLv)7RTE>#-uyC0q#n?D&zd06)POhSJC z`mLZbklyNcWyi}Z%$6d3tnI+oEORX6NNJKE&l-o1>dha~`zAgrmpix<6CM!sk@E0d zt6H=Opeio5#A+txsjegrXUNY9_H%-6*BNzB2J79(Z8b?;jG*0I;58(GD`vWvm>RUs z@X>%q-x)q|4alD#SiYB5Y-eKw?yZMtAk>HGBT%e%fLco)KtfRzK&K=RT?JzPSY!+HgYLh*Wd=`%?x=tS91JovwE*Y>-`b=YeHgWc zj(`OpoR84N{M^_;Q4@@Va~IHrzy~bc@&#gHK3#2n3%D;`UC5Kj%Zo<=v5^2eAX?-| zbu+Yvxwx&mPve{#>_p(z{YxY-vMpe_r~mF2++(nyqM>FY!n*>16vVy&Gz}6B_!US; z@6>W+MJm_kRCk&C$%+cC|Dh?&>;W7Ow@uraYfOvgYGBD3O@^?ScGz8guiZ zZGVos7GPXQ(5EL^az3vLw?BP$4{B}%fk5ed z03d^%2T15cI0m57-i5YtNkf4mV)qKfo+n!Hxb1{tC3efqALJhU@ZwQWho0vTw(^mM zmA9m*MV6Y#-nHqN3&&I(Yk5(gNd2oVA~Ycz%Z=e}lvvvZEjF{_&nskl?VJ0M%M!{I zY4T%7=|7|(Xgt<=c-XQ`KorItLOEOw2kyl2a+EZ*3}mr|RrjI>DsK2s${(Ijy~dIMx!BB3eZSVhbmVn1W%Z4|S~_AvJ`LN8 zmA+3ri5#SwL#xN@t2@2Nsx=3gulJ8e!w)KKdn9&N<_7Vv-b-ZUPL|decw887W_4$B zp^^$?zjln~y}E&Z$fRXQOl+0?y0`b{_=n>HJ+J#qY%&ikbxEZ}WyiZda`WR|F4LYx znRn8UeINd^ukQMbt-|&uI@+fob^Z=nWmy>5xI6{=Xtf}PEyWe2#5wpq2efmp4?l_T z4fJsZjlC6Ff$SJQKKvOn|;8}=^HF#J6 z2V`kwg|^yR3>>pEOrp;kn6R+k0<;HO&LNu2vwE#NEj5|L+ zXsrfAS2%N1c^-Hq*%>-hq{0}*7l56Qq(m@)H=L%Ut?ei!8ok@wi%Q=`v2UI7uOFOewls82NE zAVdi_#gY3@4qZ<(lrg_MtvCC9Ke{h2Cgz2$0y7yq@+JY0QxUseT}u-w%fQ*?d1fZ#7-Y_h+r6P-7 zE~Hg~sZrDJuqeoj{_zUb;V*f((F1(Q<%3zOFc}N;PdMlU%k6%y3%nT4#;vAObJV-6 z7j{a`(jS}~*cZ^V8Wv?nAvw>W$I-jLVHw4k6&&m5V)TpRR;G&W3yXk048(Wa zU&8Ef>XQ!gmcaF{WrwHcjE$DbP0^A=3h(Mer~>-C-h1t&zn?cN-y_#ptF1Qf@A$lk zx4B;E`MK?tX8^9R$Fb-EufA5$h`)>vO(@^Hl>y`0-BlgI?c+B-JZ_SnzH9adymhH( zk(^bHL80V^F^uX8_4nl-HVwAyXqQu-wr)(Hdm2$p-5t;FGVnW~87$v@K&>+&eg^2B#O;;$V7&;J{GG&}iTk1KmBp=}R92 zRqWWrgd7Yh0VZC2KkFpA!jaI_Bqp(w82<;djVfWRi2%@yRdJK4k+MGu_c6!$p`g1( zP4J`xUr?pjLu9$eP3GWII+Lr40;O_M5bD|QT0%}mlPw21$1<6!+54N#&&k_1N=lL* z9p99elnX4G;69U*Pu=+QVudvE=a!;nFZC}4{W9IFY_A9UidhBv!~CS%anAm%UheD; zX7W(7w#(0=4rfD01@SW|A_OA9{G9s}oP`WEwI_aTA7W$nu&}-rDqE?UcaZ8%d^Au? zcOo{>-TjG_St33;_+a~O3vqe<>LU*2Qmwv^yot1Hyjc&)U{U1-J)t8cMqEb-Q zp&B+E)7|Cy{_?l3IMv0F#d7m=N4@mL)jFaj#ZkGXR~z*GGN;= z`*Wr_ZM2DPWiT(+`1%`TKre-ic~p1HS^~7RKo>j@><{^W+SdEt6^myNwY9rFTXt!n zJb(ilF74KL>qwFQC-53LtPYoQ(2w(j4D)4G74O?w*O`b;`b1)R!42H7#gFsb}Dn|B;dR2or0l?*C)1#Zze2y1lnqI`gukyBILi zN32j?QG(3lOwVG5mU}#y_JkY5eSN}WaFx0jhB|mmiAqckR@8m(@ryHZ#EE6Dj1+Ed zdd$o=ZciLyd(@57c<`7zkzmrY6Ejuj6P$0)tZ$g(>KJ;WSvtLW;0^DR1%^1>#1!Z8ga9uiCU7JS*z0mEyIpPWq`))HOTdlJk@R@|CZqv3#*m2kIzc-B zjVak{u}MGYp8N_z72(j#Ra9?&94Sjp(N^Lh*hbZONYi8c-?rjaYqBSwnVN6Q5pk!v zH2(23lT}P_lI%l4rik}j3VbSNnMzu6M|zLK?mO=Qd;Q-_`IZ)7O&AqV92H-%rlb-l z@w`>=lbTHYs|%VqtSQl^#q=%dM9vRoMq>Naqqt?GrRkD1&T}B)&P8QF_8s!c?PBZxb9D?BwDE~yVKk%Q{)ju<8Jy_ zZgFr_nvyKXVTPCA{NFtxoq`1NuFRp#&E!O$wlaAVf`DVik$7|_KR?aa??}u!L>9@a$Q;a@1HhSB2 zM>o23RQ`m=o%1>hC$@-JxjWC^m_>U10co+EmHP8M%RByV9ZR*TL;@c1qt_!Uz1L}1 zw8$Ske8>t_zTeOw!6-LS=r zYV>DSVKV5I`R0Z@g`q4yJu|j+Gu>r7(dpV%*jc{_udP$-ej>O=z$+jk!<$i9-TeH4 zb4*o~oqR(zV$9{>8QJkoVYDe6Tkp#I{ew9%4B1#e- z;Fj%e!CAQt;U$1?h3$?Y&^$J_&z4(}b`V_XSUo@{SV0+Ek$VT$lE3ut$d+fcsXB#_G zT{Rc^vWQYiJ#mw3&zb(ZW~2f)F^@&ot?TS)O#;uO?amAyBnOX_n`I`f);#=OUD8cuJU%s`c`C{^qHhH>1O*jk3554nXNI7qRMT<7&#(JENi}h+-C)pLHoRvWTaP2 zV2e>fBDdx931I4*SL%l3L+@=N0l+6i>x%?u1M(Y&a9|Yyeu0b%pqF@sS(%lY2`H)Z zg&K^DV8|p2oK@gk#GLr>LTQkd;D}tk%p}vl+hO1U{52wc+FvFntSmj+27Y!q8gdQt9>uu zloGJ$KwoIQ8A3{*5|U(+$=dC97w^MD(YIHlHr|cP2R%03Nq3SQ6Dl0OllO!#Zgb|H zd>v^%*nOHivfE^SQ#Kl3`dOG6mSjr>aWu+|HpBcX`jV)MZ$rzo=TBU1gemw%!rO#m z=pzG>uQr}}eD~e>ee!JutVG$2q_4t*oaymBdO;ky6xOw+C5N|yYkVXN$2}7i89v6v zk#7q{mHhd9LvRz-V^zmj*FaFr9{zjSqdlL#?OEQ+8Qnw8hw`>Z4rKya3*2|Oh42-H z&0-sNu3TZx1i%d1Aq2rX`a#|*w30!cvpfw(!vK?9Y|~w|!KV@Dl^VNxn9N{kweD6| z&oNN!M^;6TUA&BbOFE@uxb(6#eT%(l-zWM8x5Mv)T~COH_U{+*G+vXo{%$(@cbf6j zgXhHd-`x{Jgw?7#!?-L1jB5XIdy56xxYN&MXMF!_L4Aix#-o_mQBL}!W#8}Lsd9nT zS&``XX$8z_EuAecxK&L4eQ^tC7LAD=qTOAf#gHzX5)a2*UNx)xFpr@fo+gVEUriC- z70Pn225*CjVz=_$mf^j*=kfx5th^oyTS;+UBL`B4x9`{Z=!88NTBek{K*~ zJmgD_8Q_i{1k?cnU1_Q7s2n*M=ZbWrSB;0YP|&b8pTw zm(Od%YgE)iNi|FZ%x8FZnPKJY*g?+i=vqMz-P)tigqcYE_P1oC@3<*RO}djB(|OBX zMj!wBUY7X%R*(s5j>mLbrq^9sN3!V%t1UnZ_H+os;9Z&4E%9>9GI#h>EfdT4L5NAgXgy01Gs=*pEK zLs%tv7@~jVj#*uY1f0IXTQT6WYF4T5^1)!w&BMe~$fdzG?%iP4za~!=$ zh;!;nXl>s69=!M6x~ds8w;3u7*yvRp0UWb-nW=rl86^k4RX(YTC>Ra^y9p{WNV!~m zKf#Z%Z($|_$~!P1jQ=bN1}s2U0FO8DMF6@0kPm^k;!KFFExOR|T|%MpKQFAqx2eD&V!M(>kYdz~lDGEbOa zmk{fFtJ8X)8OT=S%!$T)`gZl{@XNBZu~AJ-e6+S7QVB~bUX4jo_*f6FW6IBDZc>eH zJ|qnVzB&Y50GyKXJ^ri5-g$DfF}@nj|KI285vU-HMb<#!tZAj(mkAk!)ReK~znFS{ z-K3yl$i^m&If|k0V-4#vdsWkft`m1dJ*8r1BTW6@s>PF#22N-vIAD!{edk9=E%ZdB zWB{Rwz(WGY%@ahQke4?y;=j-LkHk7-|89FJ`99dy#6t)|{aR5L!nI5{|J5)IVI@eg zAa7h37_KC%FxftSjF_>2krfvg6LBO^cRumsBP#gojdQs~^lGA)6hwDHSpZU|?BIWU5Q8S1DFDFB>6J5EC7+fu>w{)vG!Vby#40(-@@*Hq1p zGic2a>I(BAV1$C-L)oM=2YhDj@PWg?6-YurU;uJ1jF14lPeReu8Y)!lFeLU0RmV@f zf4TLtfW^(ttpaH<|It(j_TG-p!qcfW?6G^Kml|JU2*IGd889Ks!MM|P%rygKQ*d`;{l--y^<6mgq_ z0F1c-1T6y|BSOJ~OxHoH-4Ew3HLI){DqxcFZ@L236`=iK8SlAz^(uHg!pCbdV^uxP z!phoq1DAN@iTT0FgQ$m&s`5d5v|p9Ks$T-*bHw^MZv-sT2B&Hv16NKRKru>D5dQ(^ z0kC1&m!pI4flL!3ldsZd2xoA#r?u-h}CaW34_cp6-8U_I!T)#E}$qZ5q_K6wUsf=FX&=hSFLZ z>=hhI_|k-<8Bf`jIEpl9`Bmjcle%L?F_kQzFHwf2%a4)dLxKh@3$lSh_D6!=wc8*S zfS1Qt9t&%4u<*hHLivE(BW63Dx`#UvsZEI+^sLJ)qoyplfW@3zbnzNPK_t_l)zvXs z0*uR-gKZ(y39{UjfqWmEWGb>ibP# zMt1;TcGR5)yw@2L0SE^U4#FLWa%dp?v0MGiTkkvzJYLOm&y>SS2O~hFCJnea2+%Mz z6t}txnSifp|zt>0N zSfVig=(EQCZG_{BFmw$3kCrn|xQ}zKtJ@OK?DvGL-hR&8pkk`tl-;LwL>S*-unU_6 z&;e_49az1P0RVqqbQWS7oa#-TfbrOU$uoXQV8pL&Y#>sN?58UkFd-`jN)DIWu zCrHrrzezM`P??*qC5zvGSv?5zVlWrmm4K5*FEz3Y93@)y^D4>A%xBZiS!b=vv6< zXR4EUGn1hI28jOUjs&oLa2gy!`O0vUdlI2qaZi8`tK>aenv>q(PF2T%svSt4c-0ncs zH!l}`bjhc{{-3w+;o{eTi!=owo3%}?;E>dLgxJSJ@ku<{_LFW9aK6IzrV$@ z5-A8qN7#_+%DMisBOD5M6ob(`cmmw{O#!N>MgKQtWz+|cr2Qo@g`Fs*lPVL|l6Ip;jJko#LC>gg2(fDtW5O>v zr_0`BUp47xAKvh`_(J;(=rI2gC&9*}Z)9YosW`q641&ksp~1luQ$N8PM}z`91Nai+ z+EbTK&iKN0`ErCj!=9Vx54Yv{zS~l%_?5$pPviLST)!0Cvk0^Cp^Y!UM8M2gir(aF zgRKRK8uRn>DM!gX>80*2$}AR9Pz(|RGz`b9HF%4d_y3NOC&YjOpk9lwFdXeYDVCmv zU%D)2RZc!IlvLK;jv;V6THL#`xFk9l!91mN2pj=K)9V7~2v8KRCG^b$vkyBcp|2g% zdQLYV2bcr7!8|*+XXpF^J}(^~Lz=$0hGMoru5-WnUKRayi-)(+G%x7Q7RpMfUbH9B<@#LyHiL6nA|i$( zyD~lBiHkI@Kc;Saew5x0!!Ao0dajIXLpot}`M zIFcS4T_B5dQ8&YOtFW!DjpuLUeXv<{t(d)Eb|qnQJ*SB@4I13YBfA)(A6Id4t3(XP zBMK0zTSJ5Q%y(wU6UZ7;lTm;8om*XgG|n$Zh(c*To-M<4*#y;g9>{$OM{99z*yP^# zyQETB>LF1E>^ln10!6E)be16yIRixyA`=2bXgcuypvMN+K-WITG{WdE-d6+20B{#D znuj3Sq9^?5RA;@%*JSiqUJwY@^ovsjp)yG9_Yw&7=1bp{BLx;vs!&RlXn%mI&J##1 zr%pjZ0fT(Y8X6`l1!zQo!UbdkuzY>eO=%5I++a@p`LirP70u$uSXl7RH0QHz;EvwrX}028M+3J%GPSC!wH)F?#1&cnxtpp!E6K z5C>kGus-3if=M`p)hxn)$O4Cngp1iE)W;;RqvT(L2MSA6hL5|~X=xnmxUE1|qNT8g zJnq8`34ttp!%4nZYhyi?Qy*s!pN=`?_ZBQ=eieve|-5X9^OanBzUv4+kZ;2l>Ufk!C=7cFufk2do%O% zgF7^a_cacrr$=#%3{B6=Rhx6d?^ln611 zw}o;XIYZFV`mIK83XYhUrHbGdurOq)fJ+*1l=SMH48JJ-j zplcf@xkV5v{{c39;^6GxEe+AEh};i(Tffl5(p*}3`lr;&!xIl^HFO@`gbl`cRn_7a%PIBq|_) zA!=19`9LQk57dw<4j6N{t6Z9gAo`_TpWg_J5#iIqIt1+-7>XkL)!v_kAPD4zDA)3q zV>=ZllH1LD#I2ASy9UD3-IXD5S9XC*Zxkl)K#_vg<-2KbJYey_&C_!iV4Uq*%{*>hLX#C-C&(;C3jBpXjftCa~Gg0Gfom&UHAcI2wH6*yH zg1Iu1n#0M-3FW}ftJ{8&AW#7J24HRV&;?LyX%S%{`3~d2v;@I;5XS%#?E3am9ZY3D z02ddrY6ns#LeuVnED3};RpU%RBhm?!M}%qx@h%Q?4bSVo`)9oiI9o>^11=8N5%B{= zm8E>U8NAS5L1bjzH}7cRs{!-y1ITDuK}bLwzYD)@7usbc@dn;8zM?@bXao_y8^Rd9 zeLEg_2#ARV;-G>MwjzuoH~R`3U_?Z(vr5;b8(-2ID$ZhwF=qM z;~yNMJ_YJb$EJrh=LA2Ip`jr>)$7l0`z~D7Q-1g=5d|+SakS&GaJ2%^7N@-Uumau? z#j6cUvXi7KFUZ9+q_`hR{8pNmy3w@%ZDAvkM?S6t>m=Xj5*tApf!66|mbO3@C3~I% zv$eRJxV$QaHYwTlB?03PJv8rj{49;4k5TwuuesqaphOUfHqyigZX(St zzBe(IE-4Wwbbgl7>c!KKr|yb7PcOtOdhwbS%fUY`GRfeH-Sgp}XblD9!lca98bm~d za)fed>X+0BnefloZv`70Ue=l%tx(OgLR1=X#ti zDrrW2u9qx1VwOui!MvcTvpbvNDp~S^Ccn@r#;a;YEByy%E6eXGvawZ|_^MYm0k~RQ zvty=>UbsPpB8z=spfB&T#xcnEAi|gFW}Hnu7{>Fk-?JuO6X>f_0Z{??HbIk^|KMi| zPJg{YIf6r<4GTgvkvA@oBiFP5{P z3@@DSjsVTA3IK;;$qQeA42sV5zF-EHzi6+0b0VIsO#(imA{0%Gidu^$rOFo1T{Hu%J|^?_60#N#*4bT zG-x1^z$d6j@4v$FZ_~Tl@;bovT(s_O#vf?uaDi;_`tRU%U5JKX6Jf61#Cst8O zNhBa^F&d2^e}`3_26{_GtqC?Ot*v>8@CPosT_kz|bduW$0T_9*@Oa6^e{*^E@XgG37rGNNk*RU^6dahy;R6Ofky@cKjP|jDi+#-m~8-E8Z>NI zn}AND*IEZ%bx~1~N&<@WFqiBeBmgG&@iX;~Ly``>)Vn~}10G6pTmVq%03fTLme8($ zAOHLI=%}HGg$7SS>vi`+oPLvQcFq*S7v`y|%BcKTSHpry9`nryuf(Y&$m(1t_IvyK z=Jy3*)yH_#X_w@JZ&}y0zOP?Eo2)Hyrxhr6Map89Ty>D^~@^f%-aIzO63QQ$H? zb$mw`psQ*tnv5G2qnT{EcM*3yO7+xOhg4s8?q|0n-N|c$E6GiJZQPZG0_J!^ip_URGtIm6tXL@9bg_`#~MSiCc z++!_OdpHrveCPO#DmW0edD&%!Oi#$XJei?xt?+$p8yMtT7coT=G4WgiB3#N)X6*P{ z0SwQceOEvc7r`F@jmR#gf^noN3)J#}xGcVS@!}a`!vn11Pvhg@S8^F*x-3B#M9K5F z<~*Wf!31s_#F8Iq^*bf^{i0w1G}!y`r0*UatKSd11%dRs3LD|y^$a!?h?MQI-}<9$ z_b+;o01g)*tbYjTfVB!iAqR@nDz77TyPtK24`5u2012G2V zOo35^I}!>JS}iR^2JVH3nV|#%1u|`)MH0{=Bd}<_o_UE$EZhO#p@c_H z^`YNSR|JtZ_7v(#q=iAYwlCE5MWEt@TVdKdFK3i(8nSC)sqF3VyF#O~1GE;vnI8Wm z(F0ZnWe2E&VZ^Qu@fgsQDnlm#t@|t3gJ5U@+LjJLMf94yct9ozXbW(NlW~MVR#Of1 zc?evo;?U60==jkU{dA{~IZ#un4oWommGHulC=|dRNem|uA3V72sG-LOQYf4`!KI~e zc~{#_uu)M_Rb7ceEW{A>7=FCt3%7;vB ztzaA99Y@mZX5rPdpTSP1MceJpb0P+Lg(^ieDwT)twWx8DEaFJ0_jyEU(zD%G&=sRJtLD@ztxw2pTr38R25s`UQoQ>k&Gg&#h2*HtGe30OCNM^x1D z$q1#g6=TvXm+nqZfvHq>@uT+v;YqN?Is zGF#3`mbF}^%5T#xZ@pQnQJ}*~82sq^NWPCVm_G@Xyx!MiSN;EJIt!>Q+wALufJ&!y zNr^~zNq2W6jerW$p>(%|bc2)v3J3xU(jXm*l%yb15)xA1x!?KztTk(9&CL5KaK*XK z+55N0=BJ0H0=fF-LTnj$IEb&QPv~T(^w`4B{qyHV5+GCjK}Za{&)3DE;%8D`F$j$< zLVA!0bR?#T{-%t=!cxkYYGRZWrp5Zc=W`hR?@APS2ha^kKC{KHu+dU;dDVrT@buup z!Da^LJa*HDIOGfm-|+!pD8i985YpTOah%Iw%oV?VUFPeW)+{K4NZ@tA@W=7Sx!mhP z4Cy`x8}OXazbyO%Hv(Wj;Z4$StAR+Gkww08DAbcM#RSUTCO}jmzRVBs*!r)b_cP5D zJR~cCFR_7&3AQ6tl&~+M4Fwd&^2*9(cxPLm9c?4e3y2*|LTteBf`Ya^#9BOuHY{}d z_|;{B3YUPCQ;^FFbt80*p#DKZ5nv%hxG}P%;TpRGe5DB&4pLPEQ(DJ-c(_-8SjIvg z+7ALRJYgHa^r8%|!mqClZ3G|IJh0-;LZKKr3MnYFXrG3s>IJ?!pM-uZBw) zVGuw9w*!AnsUxWV)tngRwS8c0hTzUj>Z9Nr@L%5LiR=XO4WYI4?fn3ueg_nDqET(_ z>&@I`Bd0#hjcUBB>!mqr#6#)1M{!SEsvVg=&9k<)9+r?!r+=QLLNlj?MXt$ASmlLY zcdG+cN--XN^kwm8BW4IlVB4MEQ@WFd@{mgtp@mlb`)H{nNYe(k@QFD{HdVq z5t)aR#cN|ZQr%Z|VM%Wye-qsk93TvJq}d`(in)f3DKGm~=jSVH%=oA0PoL6n(yO7j z$agpqC3WdJaw-$O&5(0afqa^olxR7f;wG-1;Rf#9AHtPhK8ri9S><=@cpLjvT&pQ` z)6@uF8#9gE)zyF8NvAD5N@MlV6x~9Y*`R!bn_OxC3$0{QSWgI6i*~>O{Vi5PdutXb zJHKNF4eeMu>vL373x%Rxq^6Awn~_P5UuI*RndLlb|Y$Fb-=aNHZ)plf5BV^d|X>MInzTG@F0p^De5?l9@-@7YCP zZCvM40qF6O+6swJgVTQha6;34!T;D5s*Odjmbx96y6Nu?PpIHg0s>h(w>MW*^pj`&LvYE107(Sl}uH%RQWP>1J-2h!+}kbZf8-mmyTk7F--=zCT}P zx+mb(V}^V_KRtlM86IFL@ZN^fA2H8Ey#mfFp!iTopLi_dk?ZcWfr$T6byEa9<%$y} zz94)@2zWdF*4Ww&cCExmw;|_d83eAy@LeFib%cKeLns266+C&eU2?_==XXpD7P6`h z+y3bzCDZ?s>TKYaI-iidAVzpYv1D8lJ8#~=lr$nTvI%y^JCmvvl>k&~;io9(Qx!Ee zBW5meOC!`7-V0W89ANw*KNBdAnL+8&PYrk=wMxG6@$n9u#QO>mxeWJtu3n36^V)St z8_gbyuoSfMxp=z5T zNkHvBPe~M(L3CW`o!P%pM9GX~c0Tk1lA0`|Mbb~#)J?4AN#r3XB_89!^%dl`a$Q$0 zX|}7G&s|#qyoaXL0qUu7{#<8TBz#+rE%%o4byuX<=dVrI%?-lV&=kd*9W=h}nWOWMsZRHMTVv9J=^E z%|wS}!2>^26>f?bSe8`)ngAIs7@m+ABRI#5(dfWE`+w8=UH{!@$co`GZFtTk7F`rI zWM2$cL?GBe^Of(46fY9tKz#i8eC{1_G&)z?IWJt`rRC)}gTSBxNKr~A*!sEVrrUVgm}=5^@= zG!GiP^2k-K<0W5J9Cn(qPF+)dyV_g5$@lQ333T?z&1G@_6<8jXY#Zvogu4hFFOcZN zPufly_i+T5n4DYywzUx@Es$xp~tk2bw&dI@&%bX5k_%;+&b(Egm`> zPs+12rnGX>y>RR5IC3yw+?4q(YRJF()-3?fj&{J+B3+rNoV~$qN1O~6JWqAkQ?s9R zks3ug`t!J_+07-AvD~J_SqH>G*}_h6XFk{!5Za$FDCge zbLFNefLgm+61$um>tNmvRVFVgs`>=tGM%o)sF}+4Jbwig%MD3Uv~H*OwUD-og#`$Z z0V-lMKo{BU*nz1Mo|9m2b^A4X8!hO4ug_@s$fFA;$=W0xOWbg;%>|roi=Y4L%yHvmeDQx; z0RN+D#7y?cKs{A6#78_~VOMd=vn%AeS_mU?*r?}A+k;CrgI?D*{-efD3yYvUX@U&QDYKs|!KH|$TbQg+**WP1zef#EPWc0hkp6KonkM1w-gSh~%n9|~$&*3&PMSs9+ z8eLSxfH?W#!3o+$IF3hw7`QHZDGVdBD-jF?Q% zj(5S#4!NS^n7#6lR5AxB10(jdxbOc!G2sH4xgY>}W7g!RW?TWmVdL;FhSC})a{wqn zlqjJ5Q}Fh761W7JfDagcA(V0g+f%>FpS6h13gp8s4T&1dnC@*fhNJ>|hr_;_v+& zSZuv|CTvxXwe~}lc|OaSu%rQRJvlk~78Wq{t3`(4y~`si1#e#nA&r-!|J-&l+h&J8 zr3fy2Iq0bO6P;lnHz>YN~uMgJENF2lyLgwfr`vj2_5^eLRs6Qy z<&u_iTPsbdauZJJG7@q4L-^mYZDLSMsh{mFXY@7NvbZ5}0Zel+lTsT*-CGG|vNtaI z{^ZiA$cGP(sp+X(KZpELpWkL)!6LEh7=rTtmEg>c<>%jT4LN-J zCDkU`zjMTWk819Y{`G`*k|X&H*X{Q%I$$l-Vh@Si2)E6?c^kl8~gsvyV zp}i}J8T;eqzs>sh@8Lc7=j;NG^ryg}+(#VGufM{(2YKiK%;25xfju}?;n|P8^8P>U zF9m+>=^*%h2YJ@eaD#swKr9Hg93h(h8I{F<2mBT2-%NjcCIk|qy6Hh!khmDp-(g;A4@MiEL1q?7%XHjS2DgTCj|O zrlk(bP$(2K)L77^7lFdQ~5L2WD?V|mk1HF#@VDX1tiinrMjI$C6ofq;kPiX>#%UH{P@TU_W zX{k0mo(_R0>Cd0g?CL&lJ-%Q4abE>aCXI5ui0Q|WU{o?guiU3E5Jj_FSQLU1Ln^(! zt5}`kzIB5JuVaO)WB9_>;O6?z&vuWjq;WHPZcR$%DFg*dmEqiUwi;DM-)JRhSxDXcN=ZU<`1LdGJ7j6U`0^W~3d8g5Wrze+UXa zNMZ)urD5y-Erc*vim+uMGf`wEjt@{M&=4P+nCMJ|qc6$#aH&-rLdX%ABhjVVJE!BoI0krh>1xmwGp|#r3;0CSLL`+s{T;*h8 zXjlR3hPrtB9USh^VU(y=(}%n9>iFu?0<=6(#u+Dyz|iskqfS{S4Monj>YAW^gj8S$ zgAQ3e$g+m^68t^Qo^8K}4pRIqsCA+JE)JBGE=W{_IU1BF@GC<&qtI=#;kDF22eB(~ z#^#+z(4FM6(WaLb;*ZOAOc^4^KVE$gWf~*@xX=06pUk@A6a|mU%m**$nZh>|%phKI zsqxwP>R0FNMPH*|*>cLL4t&Sl(dJ!(cD&nKQN%@sc`CXSQY6GbqYg9n{39oS9mWAESIBGZ2}iDjt0zC^x4m3opj`K4u=-Y1Gi`>B58yJ9N}HUCj$uzb|=*5J?vO2l7Y~=$a^=U4uL4t zcn_~JhkT82bmrLezhG=7dnq>=%S*S>^g_A+Hr}p@I zzbaLVYHkUvs#4*(FFm&PChXg`5i)q+KZUBo3rzaQWL z1f=G@6mf?`h33%1lK?aFCdNfQ^&CXIl$1OsHOTg#V* z*?<3~$YdhZ4};-og$dB;cDeq=08gl+;HS3c2a@J@VU+U_A?=>Gs7=QXda&oD;k&i8d_GHhO*MN9XEN z@!?v}{h9o-_^$00O-EGNau$6>P43J1TGN}Moa~rZSr8GC=oAT)5$LCBludy8!}h;c z&qSY=(rqF-$*3geVDZSfZhirXwEGIL2BI%MH!4`MQdhG|Iik4dK_IeAjWf2b8n<^9 z!!Y^MxPtA}hhbY$mIPhJZ751c1^Y2S4756}UW%7tuC)016&5lMex{EmWZ*e53YAHT zPOdeEtD;$KhJyMRV^Y+_*RcrJn87UFXr_1?QKw23aZ;+Q3o2^*wt}ehjw0QF<-#7> zLvQaD!zZc3Y>d5ZG%5B;c{pPeW8GSMz2;^b6}{bu`kZ6Y_{U?@>xyn8z9d;P@w@Mwq*^VpA$)I<-+O}Y)cXqgD${XH{T#XmoIw z>t5FpA+L%jNBdI?p--*oO|O2Etc!4CNg&mmTBqSprU!B*o1y@|L*y&aW`^ACi{pid zFk=&@PM~-g>LU)CZ2Cw%8f$1vLS5$C@g_Z`kSnUF51QXdh2Y>CUXplbb(%%>*1x2`mLsjelVa;iPi z-aCvx{9mkU;rpHcIhX!&_p_N%(#(3FRGz4>mKG>|BE`OHsLM_z&@u&g8rxsLZnM@I zfeE{+-?3-|2jiA?{~niT3cJ>7Uh`IHSP>tAl6_zwbFfTfW(to&vKGHreVF4$Y`iJG z#Uih3kJ%kO;GabhhUqo&rm!cATi=qE8QVDO&eXT$QE|@IAg-GH<#s)SDNZ`CUGL)o zx|vARU{%BNpzFl^bOL=4UvE6CigBwa+uq3uy)tiRL1Rk_wKwM)1kPw1?s6 zCG^{?D&7#4#meyl>oVY5beCKv6H4%BZrLBwF{%R^{mM$u9q1B&?78PrN%ud>(w)N6hPB#x$}ozbzF*pO)c1-TqpO%L@`icQ5n)u4%4jjmIA% zLvP@{eYSq1zUxb7}`bWLH-QcuR5X6HK?bD9!gpAx`u zKf&f_%DB0(PO&YE1aVoNjX3m?N`gMp_xBAym+KEH?U5$Z`qwgc{RW}C`)9JC^@Xr0 zuOP`(EHk;47u0u;cj`OK(#pbS)K}&QRUD#>Hh0)jlP7TuaQ`Xc{k0zE(tJ5fIW)$v zj579EzFq`!Ky{W6{P@v1N5cF;CWBK4qZ1XJo+I2&^y}9rWQD7j2N+5?Q8;HOe5gB*^X>)xS=xO8XD5G$juyzBAAu z3!(B(^Nu(^goi~DPrfxsO&xvqEl`zZV{2Hey1aAo( z7@uv(A3pI7Qf%0w&-h!HQ)07A8?kS=mw)9i)_8e%-T5Nl!hJSTLQ~sEGVtsm6=Vf7e3ATE7OKYxNjtPA3NEVl7dcL}F9c{*TWN0^KiYOv{Y| zaVS_P;ZQ%@yKoY02#gdKwExwH=Kh1kXWBu}J^K82< zubteiA|qKUL-Sl@ubm&dX90eQmCu^M$)=JchXu;DrA|~L;}ARSrHOm=*`qDh#n&UX zuAvb(DC3?f|MSz;HCDDBWMlmMEB1k09Y+6}sQ8+SoHk>y@-aN5A{<)UPg-rk-)Q-J z%=4!F9ReLPT}rZjk}4v7f!xcbuCP~>v2$yVU$-*wmX+f#%n6$#iB@=BnY1k*G~qU` zqzohwhDGbTce=hTsierg^mx|ua4}FJsLN9HPU>`Jsay2WH$hw=JrM`|L0 zNA90RPC>~_Q5kJ!_jM9R?4Y~pzyI<53H!13j-w^IizTCP6EmyvGTLJ9l50l z2kv@i#-+!?4(1<}^#oxuqN@}Vn)qMa{mG-8`~*7r^9Cyhb!)3so8s* zqcJCCji@l#6fyNKKS~<9NxHf8j0TtZ(Sj`Uo~kfDlL1A`gPP;|9sKLYYoGqR>uK%e z5}7j9mCS>Z#fg_md%iC8cG#J=*m1J5Lro1e((jLwI?jldghKjlb zHKbYCgXboB;yWt^y!168^08Dsj((nzi3PeL^L;(8(=)fyYsCc z6P;Kv5<7OMxJ4+alD1P!BO~u99RJhM>=5Q$9)p|vgcUhOVp)`UdQa=Nm=;m=)9z7}sk zZHAsJK$2Geh0-f$h}+$Cq`_CF7nuxgaHq4SnxK`SKYVtFHVyL{BKprVYEkDlCV0Y0 z(8b8FEbPY;U0Ue#ObO46=7+jE(dNmE#RQJ77JPf$RA@925y#EqXjtH=gLGiy48BFmP?5r!#ao>gY6iBulDd*|td08D)uv9y2IVTdtRT@jY)iSs4{)=gmk|5FF zXgorR!zUbm$CR7xna$e!q0fsc8J0cP+SmX2{o37@_+27Kg0o1o)obsa!@$ht!7y*` zaih4ePJC4q&mu)xaLG+X>F~HfzV53z>5JM_&e-Gn2?}?&HD)rMhxu79Ii*bGPEWe; z8TEULaSue!F1m^GV2Zx0ZphqQ8znxuxUDDMW&NHuuiz$8ZGRkl1^RhA;d&1y`P47U z%LYNeWQKSVXX>=hP+=s5*qY(nd~mhB>8sac2gBInEyXlro_H57muZ1lof0$6*SUuz z(y#JA2_+_0wjN*QKg{-LN$Iz}Lxm}Z8H-9tAl)@z`N3dwy!m+cN5g~kh22@dz^O;G zyH!<3rn57TOkG|#R_QcZZC(UVn*yZAJs zR??dr^v&+~V5RgN^5oaoMEZPf_19=A z(k~{7l(R%vwlAOCzzQ@I-qT>@zRJ8Gpfl>B4F@`QY} zLm!+t4pd9?)m^UQTWj4H_*8aoq%cKUfv7w=_n04zj`iX)h@=uQl#Nwfa1b$?6NnxOTPrOJ}0(cwu}(<^NhUoqkFtIPJ)9K*65@_mg&f ztCIbCp97hW3+^2)KBkQ{x!@qf_Uar_G#P!q=`m{N4{ADxD zEcCX%^qPD~T}^ZBtmN~^n)}BrF4avTO~k`JE~n1y@TCCccdwbUOd@oc&D&$jCx>+| z30G~yfC-SuIgriiMZkn%`T*0U9AjcqI!G=pN+%=qZOxMp1h;Q;2T%`Y2=9~V#}m6$ z%kFNg1uDAvC81#`{XjVi!CIBFa?_4;#8Gd==s#a$)Cj~q4CbbGwo0zXjpYUJB8+lZ zGlutn8sX%v7_+>6OKy%cQNmG@T!9Czl2!Y=tmD7lEOXMOPt5VLLjebgwAW8= z=jXS&rMU#aaI0GlPA}B!5yfinY-W`b;dxH{n2~W24y$BI@U@`XezJ;w z@aI(X?^AD7Z*HZ}R>eZRQnsUFMiWFhIC4qXq|viMzcYQ=kbC}fNZ@4MynB7kaEZM8 zIGXi~E!pZ4?-5~i_t*^@Aztnvry#zJkqj+mIV##j;$T^rtw)*Cu&7Bd2k8Tv<3a)f(Gqae?;w+pc^+N1^ayLCq5`Tn<7N zC$8n0*Id?ag^42}rKy#wyriafRyx*d0k`PiA{L_FPego#iDNV4akDG( zqTP2kj>e7A-Ke>i(Pa%YG+ZriD+Y~)CEeKW7Al{>Zmgp-vZwRw!%V-D zCy0&Emp(*dhFCKSH%06vw2l85lV!IGkwxJbZE4n47h5S+zc>twY=3gg_S&hh@J-@) zoEOfO6-iOJ1zEKF?w^$MzhTRf?5UMo)mju3x1{8u64#=CE~pVw+YOT5ICSh2G>zD` zPs&dZoi9yOq~!cREdbxBWKeK)7)o;7y|$8>#Ylzw4&`ZUzE5XtH4vtgjE@@EabWaL6i+r=WU8eF}xzO>_kg zc>}k-L@yI;dlYa`!lVuQ#Syd81TW?Sn7jo|KH#ErZxswsS_j}gu29twvZF}L!!*pp zJ{4h%AyHsZs9##K{a1~4X1K5NE?rHgzJ(X(CIv0{6d5&L3R#-OMUi=` zu?>n*zE)Fl4b1$_$)@fzYtge7VrA63XaS9C)w`?sb82WZ7c)`A(v9X_r8#S-=IXh5 z{+lOXix=$V&xD;%_NSUk8mP+{ithaTe)@y%WJ5D>ux;!6fY8pTP~ThAePzx4YQ#N8 z%^VYz@ishIFvODzq}zVN;o(`I6xFL(|_i<`N%CGD%zN)2r>kC5K zjrZb=O@b_)4n3#FMt`oqxwrm;%mbChL|jFU*tMDZhMaYZ+=rx>HLV?j{zQUM4N|&{ z@-YePQnbVYuYDU;bVAKBAC-@uhhZ%ny6V(|sIu`BR9H6l=~{WaVt;m&zTM$Lr;sPr zR58Vnn>&5aZmQLna(48#%`Pa59Y5rUrmFk>d3$C<&b)+yBx>)Cy_*-6QN2$Ti9V{; zvaz58@faAlh@#~Pjw7Bn|-|oGtDxmJpfA|+8ma*f%+dgyn!OM)S)8JXUiBlh!)vkr_$x@#gZBjII zF}+8E6rLrDc0ALepJiO}$~m{4&$42fcoP2y@2{v`$%F|bPjony4`d6jtlw+-@OOc3 z?Mq$RlJeIM*XE4(TiA2AoL8yvFESLLpwd43g{_A{UFPw->Rq<;K*zGM-wUfzD!g}X zz81Xi(@NV}HR+f4$hWJ`_D*ZHc&nNvA!7=yF8HD1VLZ2hoya#HbmcotwHcD?Ve8zH9m=wx1j z$BZ7<1*e?U_O9h-Z}tXhOoqton8z*RhZ~Q{$(y5{xIuMP$J$9qzF zzb$tXl$m-UVb+qlSoElAD~@-IC}WKDhyR9%!8c_yw@tE4!?j|E9lN==hctO+$1J-$ zCSP7oxN$SiZ<)2Fd=7~<_Uh?cA?rp;7>)G37)LnXKwydZpb#!R%z4Ayhebfc1Nt3H z&<`9Q9&&@&5Fx>FKbfi5t#xRmzX@r-D;g=UD%l{S^E%hZKi422p`^Jv3(i7BHzw%v zW3bk4peIhrI*SEB2>ByIgGr$6i62=Pvwvnph*mOD9dC;*ZJ>2cUanicKh-oQ$|=^W z!HXuRO>BGdm{w3MhIZq2gA0`cgP&zD!_oL%*6Gto3&Q8+?(&l7-*sP*Q2(4Dv<>$; z;H=lWry%aP_<|w%9yYI>`ao??f^gkirDN;?bhu1apJ}ei{8lDI=}ogA5HCs`XRVcb z@U4Pa=M5Fh5^8R+&zhd8pfUZg3lCbu!hm?C=c{CO3xJF!v9!UGeP!2~e=;AtYxTM* zySwhrk^x@-6Q={7zk>&vl0TXA0{=?-ZWnbw_f-~21~>pqQHf<;|ILVmO2__dAClvW zST~~X$xdml5ccjaC@z5=RYNh(Bc6%=hBzJ{Y4*&nWgq+H#iz}zcc~H$N{T8mVg2tJ?E~>6 zIjWZjk3>;T-7L|bISq1IWX-`NlbmnScQ^aB7D0cdS(jnkk3O!;Ki4HMUGK>&5S#w4 z`|$UNm=P~CK1I}_ckR*NhrUkcA}O3^Uu-qQj`OeD&d)|3*lV`+E7DbqTV5lUop2@n zy1%H|mJpb__Bdo{gd)EsrN-law`to2{y)J4M!iM~tF^z+FTUB42&e2CR6V=l(t0BH zLsgIOG~^zB6w~#jr^Hu2XPO`12MD3qx_9=)6L&(60^$5w5LIFttobibL4he5W)MtnXMlE0BB2{ckIfZ2pU{%zHl^oIp z4iM^tIB-Ei_7gN>%8FBnTn9-?uu+lP_xptC8j)n%eLMve#G?#m-`fAuU=k%5j7j8Q zKW(9Z0rDJ;I>B*r!+m?kri)+x=JY{3FULOdx}AJ3GAKi<6=zI5_KY{0(5TmXO+?p6wuHWv~*1lhI6wMkF zMg0!4 z6~0RP` zBH3bB8~NoN$=u6Uza^e$3oSg*RJ5QQ!o@9bpPuuXqG8WW?!HuieY(4FmEhf?`r))A z`tkDvd4pZ=w+_P|*N^{y#@;b@qw<}lqEE_Ez*51@)jDBS$jtieEghcccZRz6z3xT}~@ zPraKT2dA@(8&|E9Tw7e57*llCbngu9J57S_w+BVb_FtNW_CBA~IXF1qxjkgRCt#PF zzMAsIYom{5MBwUNSXcDZV6Iu%1T*hhZOwi@w(|&Uor)mWO3w|&F)PkR9D}_kv7It| z@B3BvF}L2wveH>-v@Yb57QHOOczz)!xrkL2gt0=iu`4R>t9E}=hc=@&Do_X4_91Dz z<)UT%;;$nq5^v6lHO_*!VuMmKn^t-%L_4>d!5ZVSB8j zC4m_zEpmYK;z>fM7-~YHf9<{W@Tq|C{TWfp*=L02?jMSti+48}f5*FNS`K_URW{=B+ogvJcV4^$p!LAaE&s`uiKC3h$1lW(<(=%Uz+9yvay= ztkfVzOCnF96N(NoIXs@f^;*HTfrM5gjP$_s-oQWL!==^#U8UE4pjVXu!e!PNibviK zSpelzN=_Zy19Uog@1Qk|=K13~8G)hxE%PaGs>lw@Co{oYd6Z8$2Eeai0$jiHIrYu`k7(To7M?g8^xuKlf**h`=!r7KvRb^;u z#Gh)gKb4{J2s`(!RVWd`J|rLKVkb5-ilg<{&i&y4m)dCn)%wU_j~!j^&q8%`*UQ@5 zYJqBWM-TQym_H7$nBpdn;`lL&qVeP^%}EKVKTZys;9RPY_Q@ei(5~z*bW%>TW!j24 zsURtw!cF-$+Lt8l+#;8tC{e|z%rDSYy^Sj;FVsGV!r6P%swIIPYv0{f1=GZPhb1Wx zV?Dny+tA`4;}!Q25s&)gcC0my`}z2JSIk$Rcp|kX(RFPWnhkx9e$MB3PUOz%?G(8V z?fRSqN)CJfyY=zdLgr0%@RKW_M10W!a|9I{!7F_3SqvMDd zLvMY6(T^+qB2-uS*#iNI^)c+HkWZ4(l*N=L2N_)&iJ>Xep~ul1JpF# z3>;Y{m-@t*{2U@1Q=Yiy6XPl8Prb>Ow1k>YG0L8nw*-}cV9nA}Z}*`=BUGj|VU*RI z(;G5Y2+#B2zjBA^NeEF3-)(rG`|6b8gKYjM(!>2n9{(^>rD`j7G z1dV%1N|s_RY=vGvWG|dUaU6A0l1|+t)#DYbp`PdFdw?`cW5?`A^6rC;m^iW5T_z{E zh_x29(jR{m;xB;u0hTOygRg!5>4qfk+Qguol)U?*1XjJL5c2j1Aw7UA7Lt9XAk3x3 zdl$4YwY9Yw{EkGliIH*)(%^;y7cH=4AU+kz)op6Z1d}oZ7xzWd1w{Q%9ZNz+#u&o@ z0bcIw8bAXA$_q1PJa53c52R!SgYgV>sSrj4*2zrx*yid3(47Jj0?|ePJKTUrPObC2 zDCqgiD=MsZ=9)ks3s)oPyB`8sO*mHrly9kdkzgjSg^vfL%f{wrF~r+}?H7Q^J-)H* z1K?}F1POmE1YRJzZg9UBvFaEbD{E?Mj^B8pgrqFPw*hEG_-N3~fhE8aOfVoJT=8Jp zR0h*0Xhmh9W0RzW4w^r(jrEO;fXWUekBDs_q=>ND5z9Vkv5dg74`LI!N; z05M`1CUM`;kh;Zdh>n}s5Pvek4*a2q5AVPa9@357SI=g4WQMVbT+%*|@@&D`fa`8S zA}!SZ6UAk%>up39{f`f5RgHX}DMhO!%*9GwjDNIo@682xOjF54Vw*m2)L}x6lS}nE zlSs{3_CRm3A_p+p%E2kQkaNyUuYu)7?jO9t;oZN+=F>$4QdomVn3ud&i+P0w)Rf9u zl5jREh)IMm|GJMxxrB;qgFfP3t$C{VyCkk6IE_cjBJi3VMYg(CR{>=^ zeA{wqYLl%GuB%ZP(o^NjqIN3zet7yJ1pOm}J^#k@uM9(P0#2D5aM>+gJtSMaGP?R- zeMI?T{FmkGWc=%#-;Z$qPghhA%&i`-bx*l=hPc%SquM6s%jw76dQEG4+`BnVERk>Q zOXOrBbar&ivo{!9&bCjh>!CL;B7U72^EFCtx3UxgIzd>dbr93-dsrG%qy0K;znPMv z2ormiyHc)T2MH8{i$In>T?%-JG}&i9LU=b)>}`C#ERJ8;T?d>B!hPynUwWE>MAafHR}=(Y z2QWeqizlb}U4mjXH8u4uAUHtRkN(-a?aUlNT)hyjJOWY3siewCsN{bzOlx$e7z%8s3CmDskL2g78?~fP?5h?>XfPVshlQ zSX);o0xDY6?_j`aJO9b`U%dKivMPam5=5cF(~uOjlBWn?2S8PQVBBvz>E`+lw1I(i zt}=k@dsh@T?h@*tlNQU*&ka=$m$@G z1gi&)k^`WhAnZW8AovEcvWI$r4ht+Eu&5zk3YkIzaD%u6p3QE}HYS2uDG@vxxMdS? z(BO|qoUwzsbp_CBg6)=1NA>!%?q!xnmpVqqGeL{*!dFQH|DP28%>uhw?i`}=AiO={ z!z0;*!0%>y+fKK?)zb{FQd`uExTb1IyFH}IEPh?JSrGV{|MnzyRj!tShg`)b#&iS` zb>!$Dp3=0>lX~^T_HjNWW3JrE?6){vta!*DMLApXm~c@Qv{=~D3z~hxg;o*$iFYEtSS-rgcrL-OLyp1{eX3>pARaA4v zJf(uf?}W*P-%9Tm_cV-k#StVEix z>!w=4esi&Yb`Rs)hsO+g<8>eYEju*}Rxfkyy4-R4-QNCq?KtUX8F-^ZpfG-*GnRMykTkoj72m*iG@PO`in2hyTo*QoIQ9qeIx=l8lp z!n0anF#(DJnrVnp9|XXwt!F!(>j%(A`*W6}@c>Mcy(>!y+6mO;YMH#@;7bUbJD&HZD`MS8HW2*B1Hk@e!}VETCk#AT$8jbkWDQOMaAB+~O8z56 z652o!aSpW1GDtiHC+UM{+X5^aIeoC9KK!nHjbzf^Gpts_Mr8Sbn}Oki5rAwYPy01b zz<(_-PK&@MK}BB+9u}nNMPiWAr2)_iAEM;y>e{Rh+X7)KKp>WmwsyO@_k1_vQU!=W z4>+Ne4Gc*COKLO@9Rvg|2<{P+7}N$lwr?C?8K}%GDs9m}b0?s=v6z|Hf&)vzs5 zixC_Wv^5}oMEB-lF@BgV%I=n6 z5_OH+WKl?E=Hnk!R_5k(m_sEDBkhw!LLv_+v`Li8uX7Zl&r&kh|7D7O5P5@NaGRn) zva-%n8sI`^ziO> z%;-0)eKFP)ogK*6GCJeH{+(~Hg=rPo!Z;8M+oG`6vq9&^K(?gfiYzpE{vhN&9yofD zY(IdOO@rGVqI`=0#?sIrh$Q&JwE^HF^iXLK)&Q-hxTxrwl7cNlfdh;Q3MtnEH89qJ zD-B#xAgYF|0sd=%U6bSJn3=^*Y^Y&#p}?{Na3tW2AfpUyoGfY?K?}RE7HKB<_xwJ= zp0R19WA~Vc-ZVI#G?bMw;SWSXBJ%(YLZGjImyM0`9UYXQ`!8#rhl3gK58`$MjpiP> z2Qv^5J49or0Nj-b=QE)Rz0{!;V+|SV89!A`5el#QpZ})O#^Ud!8Qm8VNynp1y#-_^a*}&!Gebo zZX$4dh@Xq=B_kegUtnIG9^_vQ0-#A9zyQEtL;O@o?xE~IdzCPed0??a)%|>#Pa3QH zg;-H}DQ0relG-zV zLwjI>J4+Q2a~t!P#7xqg(ByP9>8m>n$DCX$1t{-dix{Uf%_Poz4dJN4kxm`Q*i!Th zFwW`ORr{}@Da+(MI-f6MmnuuB`2LR%f8oT}^iv~g7vA414&3SvTX6@8{{o1z`s3V$ z>UvsS#Y%1y-t{%c>J}0Fw)N{(t3ZUalnh_Rqxix;hKHI%qQA55u@I<}Gd6o-pw1no z~StkPbR$55LwnFTIbjx|CKQO(09U8GoqD z{bNd(R$?g&I{O*2HA3M8p#sN*?vCv=VQZ+c{`+!$0okTjjkB5=?ojJLd=m<2R|F+Q ztLet<2kR8_CGT9E>+7pe)hJ=!g!3H%#voi4u!<&Q60;z+7L&5|c!~1HZj3driT~Se z9r+$zfp73$kcD>zlFSF-_>#s(GJX9>xgqEy{FquCu{EaES-t;$evx525=#}zc6ikA z`tGkD9)r{FVV*Bbgq3aFs5+)xbM?ycnQ_Bo3ciQKNc8p6RNJKhpavAu?7$2){@y@N zyX&Xht7AhgEh*r1-!phq0KO0?w2+-Bad~2cgbQ~pJr!-X|Ko}wtwd(2dC>+Wzs>5F zgoTS8ZzQi6@%TNe`IH9c2EvhN2vieE$4!&~sFA^&MFfk6q>TU{jD!S-qdbkxC_r>7 zACQtYQ_pOdrDTb}Gzc!gWQbr2Mscd9>CIEBdp3qJ@fA)& zu?O3if~bVf+9o{8EMCP}2wI)PRIhrG(tW4uCMgpJvsT!z?+Jw4H#(&@o!g_1ZZ}=P zDG(44FvCJh>UfoAcMV>yg_5PjU+9zJ5$&T`QySqYQ*ObS)kp1@lyBF`LN^~OK{Pcz zE-d9EW#d%LyrX z?O+8B+|HZyFs%sLX1tZ+&C2leG@+QZSRzFUPsOQb@t^8dM{(eT{xA47_9kOEA4AVt zXbS^Y7L4lNryCpkzZ_U6y&Y>gL-IH|iZC{rt;R-94D&DZC@xJWxAw=B41>ZyI}2CY zWES^hJ07j4tfXpnJ~KX@v3Q#AiHaRFv@>^E+|}V~U7LR|?KW z002pHm)%0FY>3;;7r@uTU?D?la`4TQO62dpeRBWs00=J!KzD&kq#v;XBSku(y)-kP z=yvxW?=B>kg2@H^cwntgX2OjYSN?~u28#uPSxex|-==R}Nf7%F*9(cg-)rr#0iZ&q z-Ir{axwZ8e)-#~`o(c~lJUS$w8wo!J_+_D16eNKMS}*hs53eB*7|3-02*)DQ+D5Fa zKapnz^4DMX(0V}<1<&*zguMkt!myW;^163Hn zxgs0g`Gd#j61j)bZs^24k`{|Fmx-hKBQI}l`70w!<&CSD$p^;!chWCTzsJ5NAUR7C zwk-HW^^bwkWnvb8{(gVGnc|CTLp&0OrVHq=X~|YO_}^wSB#m%{bY-A(iWiP3#OmS? zfNNoP_5ZX04?)^MUaIUOH(-<~M}oe8At+~cnDCWy*ep?wfIj0Dy{clsqihzBqe>8p zF}`?%KTfGp;IwuVO`va&O^wT)5nn~%mOA;?99=P2CEd^LtWNtIL>Q3&lsQJkQZcU9VSu*dMs!YGnWO_IGCf-b|MHq5Mj@Q;1J}enjBM{<=0_ zv{y%@t*&S5tRrspC?PFDsmxOn8IvS36`7}^NHV1m zGF2oLp^Rn9ltg696s3?%NhpsDA)!b}DD&{G%iizzJO2AP_PgJ`D<1cKUFWdQb*?4K z7*cJFh=<%e)H4X!39HjgG&hOfNe6t`F#(jzvdjpM8TMv(SVkx-4=_GyPtZ_j}S%>_GPoan%!$Z(wvFm*@4t!2NxGN7arZE{-`#}i`$4Sd zZ6(ul(v#^I7Z;O1U+Oq(e^xaTXziLTo92)AO`eCV`pPZ;*y_;s$LbK1WM#!Gx{b*O zvOtDBlq@O8s6fgVYcaf>t63l8(eg5D3sFiB=Y-? zQl;V4w~se!9+mzO5PJIFz%R!0Wr92u4Kq)+<9WUQ+tj&b#@UwI%GbrtOmJQvJkeu4 zdZC(m;Q5b>|77gu(>;AvlHS~Hqd{ZM03Y@k_<$wjQhQ(}_wJ z3O<(?jn?E`khyw(^+ZZ2@8w!?k6&>KA9r1<7&#j%np@A-zYk2o|NhcuF3BlyA3b>* z5}JknB2aE8iLNec*RH|9tan({HGwKbX(@5Xb-@vnuy}Lh_U;JbQ0p%?>D8Glv|^+m zICend6NWLLmb|!Xf5EXQ#%B7yTwJO7r~ASIe5o>_6YONo}S`Q93V|o+jJ7 z*B~@Hfhc`xtaKVxYR7YDUPyZqH|V&*g@B4IqbJabJ4=?<61BN@V+Ybr4_1`$v2X6I z?RSRwAY3ujBT=zh4mKtVo0Z&!|2rhE6d?!!_{d2{6KEk(#DD_i$B&A~j~}07>mT?0 z8~`#}YikYKR2@{`_=~97y;_{=e4G{k1xT-}na-xLOjm0>HhfD9*7fM)va^NR=%}%6 zs%<_-V)D>rfec;NF$=HV$BwcGG12_PiR0Sn5=DrqQcWA4T?- zyLpR9oLwQXp}@4Gx7BX==6>s;(2hT~u5TC*J=`PJxu;+L*kGxJ(qkPBdQ-!0KDiXV zwjB@iKd3iucxLFO9haiB9X*Ux)Gzrv`ErwQH^pxX^l3^Qd0qSbeC|Ka*Pq89NHn=z zmN+3teOzULyG=cxQSg3)(X{5-rK#ze(I|z!X`h+!z57BY%O()CtA^% zwyEs>a-!#6HwR}xcf0AXY|xqIj=7?-yK%fPw=-OYUgP58Hk%|5-j`E_zQy71AmZq? z*NXeK-**p}8vf#xooiCF;WQPvfp1lMYi-%0_<0sH6{8Q{UW@NEgY7K;*%TXF$D9+b z{lDVX`3>pXAzrqX6OfsU+MW$S(4wG2l3>1lJ*U_2)x?YiV*G<9&z-mlfud)&(Zlnr zFLkWEcgejk-(`Z1JR~g#U}luCU;w>>7Xuv7k=l(*uk6DBxC<(u1XBROZ!r()S5r7= z;Jp)1|FS@MiW}Q^0AI?co;|hBrl(Sm;ART?4NNUsS5O6t0+^ywBD!7bMU;6Q zP`;otv4q+WfIEn5%Btvn!{l24kpU1RBnmG7%*?>Z0xHI!li-?=7~kam>yoRxJ487F zi2w&DKJoB@fbKP}sqX31+Nfo)cnyk`Gm5J|=r>~Kflk%P2nO}^$YGFXB+v;wm@flu zB{)f&nTJ_PDSnCEOJJ^IZa-TX$DUGNaH!LmL<4zBUZ9!S$Ghsjd)kGdTXg$F^X=5=nm+Ec|LGZfko|a$R{Hpk+ijZ%Ta@&^ zd`x_zz*#p`eQWb|yVRSVKbFt&yD0Jp(*(2Sa-_2;&)+8Xv>t03ziY{yLcB-xn+S1 zOEVX~$XTv2HQybS+zI%l%oX% z-a%Zrcu^D&`}ADNk%xNFiGiFG?|}mcpqin&&Fn_~_Z=>wkXRxjAkYHH>$#O0TQq5R z@YJ@ocpA%BasTj^jm zKQVqU+Su<|HG^UY%5lGR5a_ePAJ2rUVp>peH7C)v75&&t<6?6%8YzuR2b zdw6%5)A`?B9mvCRoR(p&t`(ym!v|__(VQKu10UIY`onYg@MPuK#@^zawsYw_@oQSu zt!UqoTh_OD^6J)5j&-5V$`*{&dG(m7tZ%#Nw9fL8hBwyPo}-(OE6lI`7&>fxt=E*M zz*Mj~=ybJZ^D~TEe%m&EOq+8G5i*g?vHCpos{h(Jd$ziSK*pWp%LO3J?0Wy;^0Qi( zgSBU0%3tuVD5+s#VL{y5cG}YyI;yA$j=-a<#M9m7*V$Yc3w`mPHgkMAT2xpmjiDmu zp(u^4$(Tih??6&!09SZaR3!lCg`m!Ked*1iTxyTL|AfLIGJ~mmhqZEWV&|wkoL-Xm z{z*!O)dj@jENY)44hAkPBaX`d*z~q3BR$=w-6I{?*GK_0xajBYb3G+vA$9Q!Qt_^` z**Y$N1_lP8;6of>U_7El`5+}?$#{t(#i$w)Cu!>fX|2>@*?eL80|5;-mT;8?>A_R& z{`!oOcdqi8MMU0kNa}2=Ae~($+=38JD{{w>@|+P=H``^T+o?rXA*qnWI8xBWas2tU z#}_MPMjxt6X+fWvJx-S=c*D(gg5F!y?6rdV(ydajR0wRWni^3V?Mj~FRd-Yf4L+|@ z?QVZ((_C-Xir z2iW;h1oV>zxjIESpHZnW-+bukZ^l?R{_c;hpx4DS&D%cJO86vAa7_J5an;ZR=?xqkDk+ zp{FD{@R>D8CE(jhg?mFke#{s;WDm=NSXmjYbjsK`!HW#HF&M9e+EWAP(r;>_cu9nX zAs7xF9J4l}cotv*@yP`Dx3X+>_|MQ8CgXX`D>pNq5Z0#6yu!7c8;tXViP2BMJI|oN zBShxA%_%=874LJovEfQkH}2V|C26*kpI3 z>2>^j#%6Zd6)RrWn;7(LG%^4$twOWvX!~ofk0--S_H9B`;Q4D`sCmZ1Ie|*_kVzOH zWsO4lsha-qBO+^f&9oFQ(?C()@-O^ zxR4|6Xm`2FIvs89EiN>y3f2PgHrEBi&eN1IhH5&FKJ%K+Jm5OaK|^zC`~}EczTQY{oEg8X8&S@&bCltXU+<5#}Q>S6SjVJOi4O{3q!&n$IVV zjlD_$Wy5qyh%mhIH*xu!g(4QQxP&%hl^88Ro1j!d<)6wM{H|B_zLwHLG1jvk+iuahj+u9+_wcSwwbr6&fNMc zv|&xEU^;vMZvD`4-j3suX|6Bni$}i}p71Z$V!zXG|3d2Zx`Ls*enZNX^X=RN#=cMAXE+JM(jG;|D&9zhKaI^I#q z$;rQtKK!I`q(A^M}YkPT|Xm7UAAN5GCIc)!Yi#fyV z;k)CqVWY;Wg1QNo88U-!xBgMZ{50CK~*Lb7Qx?pWSE_ev-VVq*=^*H zwRjVhwH5)1OnRr1dPNTvE1zxkq_1ARr=fM6o2yPbEo!Efk&k+ItHE1N@9EvQKY~4_ zd)7ic*yFBU5|j2)i&e*){5(R*vmI)FL`LpV%prpo;<&m5N`aoSG3cvuexMaAajWRD zu>{4sygtGcp*(PKJ&~TnoL%eS+N)Qu#z{NufQTGq$DwD64i!8f$SX%)6X+Oq_mc17 zvQ5Y81MmL*`xB2J17kEuMfREgyBF~Zfpsue)h?H6f3T6ZnLN2B8=<{VMGbkHSRnH-@Ma>LTrI(D;ut!$wgmLnAz0y?BW?KNu)?3mKM z`pEu%{)u-*nHz3Zj7IX=#X8VG+s4NmtFOJ>qI14Hb5e_4Bib6&CG7eWok|R)FcGJ? znXT`f`ry9#7pMPI%k!Og4HSl6%6QWG7awZIDIUPa#tIApUKPl^s5qPF-?eO`4%y`2 zd9P$3+%<>Z@x-xq=mevgFKYXG?O^kwRC^IV8%0X;^B&5g73-0bru30}Kh1HL$(|d> zu7d~fL-H8v?a52m;6Vf#RR}?*-=v1p@632V(T+y}tH!4US41RX`z` zc`AM>Cnv{BAyU=PopTMfN~ABxAKxVZCNcE~wMkk&r}j@J2}TKM@j#W+XL-prdC!Uk zYJtbtmp?CNz2xx&Ktw+M(Ib9`N)!-aZst2+qJqgGWYho%ifL_8kBpUZ=7~?=d?4_Q zn3|paorzl~1L8EFZEau}N>N**n0%WrNlx^dxnaf=pU02G?>`Ff zH5?8l`G zuM0?xM<*<^t|#?-XYs`hOkm4J-il|bzu=j3p- z?xjctCMc)7KPqA$lBFwhNOuTpE$Xki(+_8iM%FxWeRH<1ZJ~eSeIgyg2H1||w>=mP zzfwc{McS>wVEUwx1n*x>HHVmBzTF}s>`dxVU;#}J`389J>8IOb>Tj+%;-rOH7j%SM z6X;;!nvY9)K!8b)BF`ihBfqX%Z{Z8pd|vOK&DV}UYCIZkjX7@MhNloQKFkHH;A6?k zS!PH?&-N%ND6lW?lTor>m>E~D-di2|FvU;p@m5L3`VWqbw$58~M%b)gCIWpfWQb*I zg~DEQgizgOzfPe)OWSM74pS%<<;GN!N{!M_5N2LKydCevs1bW(eWAtBKdbEX)f}T; znj24j(u#lCcZ0g{k8LoY*n@vGugovK=3H%|S=5Lje!Fvt>H7qug9BrHl^>}8 zX}()y#n}Az^{UQdhgr>Ae9H<~JvdGu_2#&;a1aO4q);Q8-y~n8Big~ja%ZQobdWe} zpoo;s9<4PZTA!I6lfIq4k~}3Hvp43A!#RUUrORC0>QvmOwd%Kmd_Ar6D(1`T?puvM zX)0@QS!R$mj;5_y&olIjlVg@q_EKZqY{SKY_p_ywOJA~ns<$*e=42dB6-=?4*s6Ui zA{CoFIL1*#Y#pvMk``~)!fme|_|dqh*Kd!{+zrIy0eOG@6mOnX`2p^_t^UCZY%^rxcn7 zCpXXDhfk^1r9}4J1M_T`rt&$$sD(oeZMQsVj2e6)rk6Lk_V+es|A~qs+O;kLx94XT zm6rxwUIcUp^hP&iw;fnh*>$*2DA|v3gr}qVx8f-C+;BF7V8x*abQ!gqg8kZt7vKJs z-G5lPhU3K>mOiE`1A#-my;R#cGTc9|_P3zk8CjE(Qq_^*ep6}L*5eJ`5jVQxD%R(# zkLsmJHC23b{uOrU)A0c+bE8<(p999H5{kaGq^!XS%}kc_ zc^i6XzV4PXb-y>a@xc3}AB$BIzE$l!`_yl+-iJAp{F(xZBM)6=_bK_;Z5zKiv{JIQM;)erG+D6%3>BRV!QrMJsZvJ(ll6J-=mG#XEFPl=A zUbh?;P(6O_;$O@h->I zDPM+dPj_`GY;JLKN%*--_K+c;#fNR%ckt*e?COcw}|9Y@Ln?6&k*a|R=@Xw* zHbxrPRT2i3#ut4a2?Q}HZQxdMm(+^FhbPMe0y;@BYX{8>$?&M9rDggTRY=KwtBWGc zFDOGcS-oEB^7~+Nx%-;->Jj^284_9y`sTNnUiTh;Ftuq@WZFl6Gq3jB>uh*}88?SM z=h}FEU_3(7P5xKV-`Kjf!409`og!pCZ&@9q$qT9AQ{$;pHFr`xA*N~-%N3L*G;>JW zbV|o2{E&mO>&<4qfXt(*LNxp@=;T8pw@0tjGE=8Z3boF+eYN+_izx4w+n3BI42@}Y z4}Q_|5Nz;GXPqtiGL|tD??o@wy zE6*xr)#7aWzIkRkE?tF%Qf4F7?R6aog8hE}Ns&DiCw+I)k2Q3y`i+?lcHQsRt`(l6 z{;XBc!%R8Q5yw`<5tM#i%itFO9nFd{UxVCqp>_vl9hhu$(uz0G|5{fpyour`$(YD1 zGHwy*YPc@J{ho2Nib0R)Dru+d7v8Dr`0PuN1fQgq_m8R@^IQh=ee>amId3L|KAj$y z*=M8khw1iqDK~zZs{#dI)Z#^sDhUQBPplcNz0y?Z_>o<^V9_|%AW=Qi);gPdmCnZhpKsSW})LK>|aw;~jT{T{XVEU#Vq+OC_uHh6o~0Hfv8+WQ-sFT|Gp)81IZ zv{lJ88`JW+zrzRIM_387DkWtVjIszlZtDrt2$Q`4{Rop3PFHXwsseyd&~S}tVS>8N zR;7;_7;S~Wq#!f&iiiLyl@HZ+zDV`)is_&B4Os^cUXO0@wn$+=a%pF5I<#>)^xGHv zQQ!q9b#Ml5^HzYvDB2u ziuJ5`A=l*Y2sTs1@YAMP7N%n ztY7Tk9hJC?M{1g{VomZ<%WHSSGQTJ3NDV8krtxxT(Hyrp$s%$nlitserEn^}y-?w8 zSDm~NO`@mS=Rlj>G`El>wfHFZjdb9GhU|zPD^F5dc5ey2WMDqZ-}cbj@1&B~4^?S_ zl+fqTB8Q4@ur54mFf275G_28W5S(gJ*W5>Ku<3fL`YG>YgK2qNp0oFMo{f8Kv*Bvi z>Q9;plPSTEJaf~MH#yf4gg@jZGyDIz06mM{J$ax8(N>=M*^&u+Q0wy%Ucbj5>VoW8 zNw4r8GsJqJT{F!wWh!u^$z2yHdK2EY9AT7CpKihi(6N}F<}O5(XQlG9HGSvp9sPaF z;zmcsPKIr)&&Lu4jIVme${%x~F-JKV5fkIltH{(4TI#uG%CA4|>gRy9#}Gdgwi@V; z=IBwPsh&N!o}JUd7Z6F_VCtTBXFhh&}t*m^9+6A~P+@n@?;tzhYB$_`{&nrQ3xT5rfa1Rquq^;YIKdE9sr_@{Ovd zP}wW6^dbutjw`05NygfAe~~oYdN59RB=z-$%CWrO+`l?0k*y+1&i-elv~0y1G8_30 z^-9_H$bHLya(A2T<0o$)l(g@&cMoG2e4#SBociE)?rKSmHGE36>1RJru>9( zZnEA?7DAWqeF$CSwn8u1%0k0&#;~ojs(3Sm#J?iuvTBag-zaGC)U5(h8w;NN4mzN#{eZvy7LDcjoXTt#;em~s?@I_1c$|D6oQ+sZNBQr zc!2U$!1=6b<}XTY5G&&Np>k$hYxc(RXAghH{ummn7;=U8vei$K@rso+Hn{Ef)Ha6l zmdvzlUms>cy7s(8Nv@07{)*K;SuFoxI$zi5W*6H|?fFNgNfsv)Ms5}Kd)`p_oF;h2 zR`H*(y8f=@cIH!+*Ltasxv-m>4QBAQHC9l>OO%J+|2b9X>NEbgCWr5IOhD~}vy?f1 zz5okGtvHn}>+1^RwjVscHMf%)m!&28MW&#ux0OIJLnXUq>#f9TGZ&Nwt}2gDH-{ce z2%@6pkdw|E8g2jMZkMJLlq9An?5$}Za_G~oJzw_Gw3(_F-#hbp;*m0qcmSGk%;>~v zd85P`XxF1i;AhzIb$Y&_$N}3t>KPPl{)Bz@9oM*b>=EXP2s%319F6CgX>Ni{#0RS~ z1BG)f-dW}5fj?58hKPjViT`V1u#JOF*O2Yz_x4wpfAn%>8#38T+kcTW)H!3-#(O~L zAd%Xio)_Kym9K_lW8{W$?qe>a?-NH<_wUVSIKBLO==Ip*GVAf>l*Pl06Pe#R5(YJO z9tqJ{&uX)1k6V;3FLMr-uF~8@TmMFE?!sdkd!Z-|QIT*9lMl^N-%1}^D2XRFn^JGe zn89gYFr?zM-Q;QiK5I(<^p9^&ml6jt!(b=!VDftH8p!bs6V{U)cC}W*)!cWu zRaj)MzI*Bm@U#B&*HvC(rFXj>mq%#K!o$`h?CV zG5H?xkzoOSf$m=~lHqgd^=IBo2*}q@&k3@8*r{GHt9z^Ui&Xyvo9TgP<+^jZ^`}0xB!2sA zR@!p!>hr9<*M|+-S?*_r4?c?+{3~(EFUIOKUxL^zep*+`oBTN{W$l-KZI2>Q9o~!I z%@b0}xpOG_-Rm83dv>AP-T367!1c4W5i+;Ziyz*OIJsSkc9u_fOR_Aw6%8x=ES;o{YKOqB6_tiX2b=QFV>9+Z3juJWnfs zj(MFa-PRc8b>+<6Ta(#`7nfhN`3_H2lq=m*Oy2QhN<#BfSh`2b(CE1 z!pWw~ye_Pk*F-fQ3k`KYEtOWVOq?-qr`MzI5AEXDs#kF_@i8jVGHL7>kz?#NzM87w zZxx?BJ{w9e;+WJba^=mU$M?Ce9r0a-f;Doa$oV+^iNRpf;{1UhQxnq#>U@VM?1CCB%7Q#u*OOM;v;mG?voOnGgUFOLV{&@}&=-i5Gy}I;u|H zrk^>oBv&M?pJ~`?+d|8>CHh`>PYgit2(>dN$B*0U@kP~TCx`0N@+&fN zt1JgTvT#o0uR;CYT%!Kk^5L#*%Fd6HCVc)smCBc|9@{^(ap=zt%5(M|+Ar6tRmCo6 z`C7*b@{VYmg)EiGDQysE0;{0RseZH>U5T@Jn_IN1Y@z@&jaiFXBZ!p%D4%@%3r!eAywH3gYz0cX{ z*koQ;1#LmH8I~-l5h8E;2W|T0F|}Ap#>GW?9kn4U<%(^4YcueEBgbX#aM>k!a8tKJIVGtPgBXmJ~I$w*?UW3;<*=8d<<~9x@ zp(xjhj?RA$7I__e?a#?E1p%$&rMTX)zzyT69}1{`KoOVq8`9u7^H*ly#>iXQViFn8 z=4O1g@Nx0h^B-p&`J1+%QgLTH!0U8}?=a8G$72x;cB$~#Pk)pm z#jL8RcQ>1T!#W9%cTck7pN)!%J>KK3I0tw?ZuOGAW1#P3|KXihlg`}hqfnBXXd78f zsfh0D?}~Bfroa15HrI~HXq)Bzn>T}kBzQ79BM<-Eeiu=suss*|*F)|gB5?8!EYl`v z+qTlLy$~PJIoddP$-mpNw%6(Hi;AK5pNDFyX4S)0ZrPV#KfYCHlUeeIb+LsRZ<(Uk z@{*6sUG)}psoE_%aut7uCc(-PrH3ce0>e{u#KvEaBj{3mF`KWqLC4nND7v^xK_$_ zr48eG#=O#MJQRnGE63pQ`eiPI5al_)KkMuQ=|eYWR&Wol7ndW){{{o6|IPn?yp?ZV zL-XH%4z#}09?z#Qq)L7h&r2w0nVWOgSuvqgguHodng5r3VO-0P=a^R|HwCs0v%BBt zjo5$~VPphfHOr##V!+0K|Mrf1{`Qpf{}sDmD+;xlFiT~xQ+Q0wew$$l1VaRvXzSj; zSExtFTxGTJ4Y0j{KQO;&IUtiK90Cq-hr1R9|Bn30U0>7hjMZtXKRT2_mlsFe0!RpB z3WBpJ_FgKBw6GJ-O!GjH29ct2ugBEomhUbR_wVo7E@N`okB_{*qq@3!x{}+D^GZim zN%WYTuTzb1bskiPc%Ms0Tg@1pdW~OQWsHh!6y!QSmcoP4*3RGFGA=EREhiDPEm7|w zjX3X%CM+-AM+vhHTpvUx!h|Oa6yUj7rf;UL{EFj6wx5vI@T+8hj<9Wr+Xr@S5i$;~ zLd8smM=@sZ9gpc*O`=udXY;glY6pN7(0?>3R((}LrXJmH*l@CqcEH6ZzK|K(iD(m% z6)QIH?>c<>^6cYHtaz zFnF}Hm=I+3Gnql-oShsjI8)(gK_zr{B%F~M53-N9eIGU5{mI2mVp}Lutg)UK(iV)P zgK~BnM}I0Wi#KpjSi!_th|@4}k$(d$Vzy^_-bd2<9W$X)V>BGFZ@oci82iDR>GEZ7 ztiysPnvQi^thfY}!>8=gg9qFNrWnlV$c?WB!F8e9d|{nA)(AYo8k(t~zjj!RmCe8b zNoyABaPKZ&o7{Qj%9Zt_(^S-SzsZgxl0n&I;+WIo-?mAz2W`_3FHTWt9+D!P$p?F8 z>5hi7^}>!|w4}uNAjfI6mz7JBp38%W&u++&!{1 z*#o(b7endy$qFZeCk)fAqUXbwXXw9N9%Lt+Lt@`aEWpc_7sn`VZTp2VV_BS^#V3D_ zRfS`~SMoK%fm+6qX#_8YtVdRpc_CTLruUhFvcQH9$#&0hAUVAM=m}`WS1m6jFGqnB z_WSAb+*3lm{ZX=)Q_3J=rkaK1}hFw#^W*NeJb*~d~Q%4=zNYPOMIJDzZ=PPWh!LvkQ>96_ny;@G6 z1Omeo#;1KxSt3joS1oqzBNn*j^W9^{C9b>SkB3q=ln^4pGY)`Tc37NmldEh4ytwSn zor6xn#SfMcWaeD_ag=z-*Rp%yGQ%R*7U@O~+b3)X-R)a0mfOV>HX7MrAMdZ7#PgG| zes@>Vgb{QyxxXz2WD6WNlUN=kmk92s#6SI^{JB3RkR%E*Q! z>w&Oo!a{b?2^kiv+IujJ1TNx?G)^uV)>ohD6?elrk}$xL+Z@~Nx$5&*me2z>?|hDj zynFB7ba=Ikdi8F?N_hEKz-?>isCFxWPzW9xI7qBd+v;HeLa1(JGX>1N2@A;p$b-NUkL((=%rFb z#vBLTN6vt36Sv!9{rWfbC}(`BsB2oLPDZO@mkW$0b_Bo2S??hCFw?KFWJz0@0%mD_ zJ}9%V=6*iG2pm!pdPZBZnEuZ{)kCgC7DGR?CH7nxR%;5bPCF8^7x&*?be!_+0$rq@?GZu>x!0dxAIA$5o*fk_{cZtO^*Q9AUdAO);WCAOQagZrIL-j?1&>GZSRp zf+-m~28M<=knRf`utG~j3JCpctVc(VBO@)1&!GOOkr6f}Fwek`Y?R)$qcY9uCC;z& zYOfr*0i*82`Z^!H(tTZz?Me(z>5ww~OK&jpQ9{JE@=QvoSKd>|7)R@W6NzCL_%|q_ ztHT5ytq9bTB&cp;yLjAlLtrIF0e4=Eawpc z^F@z@fV)bBYXJFyzddZe91n9(EHRM%4G#Arh~6+$a}S((MC(8(J|S0h5rc0o8g z?cyC4TphVTACyTmAc}NH9u~;6Wzc0J@e2GxR$6R_kpkWQ<)O(g-=z<&tw%I8>T@fK zK4#%#1JzGck6OP@-_WSKfBZvpGf|MK4dbjGa)sFI`a?ZKu7G!t1S8cY0?@FK$Lt&I zwbk^{?D6z_r+qtnRgI)HyH#|yrz!Mv9BZP|SPtT1`d z+%@T_zxbFG^_aUHr&BJVq7>JPa+N(aKs}8b4H2^+Tb;Hwx3?Qdt3rRa&5ez&Hhi^( zs-BpZq`~_uQWCWOL}><^J(2tx>$~}Viah1Ch3>ui?y~tOCsla{Wk@ekQ9OLNB*X(C zCDcHc2A4*fmTu?d>|LpjHt&=^V*)BFtC5kU$(q5bKr4v@Sz)V>?naq2~4a`^3E3vDnh& zxXnwE`W-FO0TyRiYjUYlAF5+99GeHd=%i{!-I4W=%COZ)X{CNVC!hiLkD(EJ%lc?* z>+?G8_C=H0Mp8QvP|(CapZa{^1*$(3&JtR#RCd3Fv8Kea8Ajv{-A&OLX>ud z5)iPSkaan({NN@A^;q|rxnDlJlhg==R-SG**l-tUs(<4y%j(0+oH?bacyQ_px(A0j z)9q{|j#ahuDMsdb`lh{p&3)k{3)44drCU0ose+zt@HEaGKXLY1JxLK1 ziqCo=r0-8s8S5osF-^%!aSsG-y*sw_c{>&#lx#RI$oxi1OG>LUUlL7t6sQ`Uyz^NeApvn>a#q40)2zf7 zmj?U(UI8M+!$ybNR}rrw&v?Y2l5K50cSKdS`f=jqSqp<>!Tx* zlbcZZG!!(+*W(iYzETQ2VGCU#wKYA;ZJvnNgOk?5(P3Muo~bpCE>p!>fn@~J6}t5g zZr-}ftTN`fDdfx32?W#cQj8RT>qyD%oYVO^o~C_lf0`bOa!+L~deeD9@k^#PCskr4zKHhgGlxnaQ?$EA)s|5~|^=BI5korgcpL72b= ze#R^1Pz4u4Hf9#!+#;ZX3Uz&{;v3SenHl@umod0=g2W_IYjKOb71iErveg2a6JuQ`^`$LALpKAA6W#Ttx#d#*1;8}O-!AP&Nb zfBLqN>a(B~VIigwIjXkh^4W=Rbx=@ofnb;7%{Ewd5W816Zi2~$Y_BSGFBQu|_4yJe z=TJ(e>F-9h5ZWN1JY;!qqytaaVE22skF7f&?Z3;#rlFTG7!k07C!qq?s}j@vd^MFd zT{WjIsEv2O?X6z4ULm^;(sM8N$m1S>q=uptl@vDQ`hxf5uwLr4j~77h>xXTp2-Zi* zs&Q;O)|cS<5)~EY-!n!yiKO;~B!%v&Q=1M0jD+AeDFz=(%V5Fb1x!bw3NW5xS4haO z#B>uzaL<>Q=Ia_79zA}13r@-;-6BNxkDp;(51ulBph$E8R}80bLdpz3T_gGeEL?fY zEn1$*G!!z(cnEu4^kPN0`fHJ4vEEX zzA#D-6RZ#sziQOWs9F$H4_x^ph^7`RCkxk>AOE5s)IJ!C^A25!1_Tb26J+HCwFSgh zbdByoj1A_W*tKE-4mWHR!dNq(ot^E1Ob)U(R?A>cPY4FTzbX=+ zdGrhivgY9t{FQvV01KjW6 z3HzMcmQpUQ2XEtN&z_BZDZP&yAY6|jJ2)-%knYidb8CEhp4Y4kDXyWIa24NF8PzaQ z2xzn>LJa73N#?9@y;7SSCWb?<@t$(>=Y+(Kj9#E4!968qb6j>DbgNvTZbc}ZpfRVV z-B4JhD11jbOg(AuefadL8h5Q74Kq&t zcThhd)8h|W77jp5cqCZHC)F~F9~=dMDuX1@V@c%wohYKvccE=B%U;vBKqx*+ZAKsS z`0-&qW`H84nunqYz*@eHSN;0&7aROqR@R$*VdByZiV1LEHaSo7iHHmsDym*5F%%;h zGNMdQgK7~Zy223niQl`PPXbN|VU|Iw1~Xo>HnO6KutSzfFs?z>1^0Al?75G9(E^7D z)p%iHp%W4`Hmdgj3=P6lkKjG(IoYruS~9fggW(Abm`gH_6WNXkuAzMgG$0WE=QR~Q z;^&=I)eG{w{>KFngzi5!5iugh(I>mSplc8{kFcd5VMpA$Ih0&e96C3(cNI+Jt<+GyXbWAzI&p;4jsv|xAom^S{6z}Z{yc| zKh-nGqI2jkGp+#9o+4*Be{$)A_pUfAtpCPkKEk{tx3l((lmk8mII(@>Gr3_gckj}^ z7@o(Y$H}4(tgHZh15a(%C~sg>&V5xI5qOY=*`yBX$|err#j+i`au@GpSGYR+ZNb@jN>lB>fJ_kn5#YChPfd%+SZBhE54 zH`mFqsM2ZBA1qB|+V!6pP^Cgpf-HL?Cgbd3HJk48A-pead_W%u(0~Enh(OcBvtk<~ zZ|g;`b2!9!+)=LPBgMO2>Rx=XX->j{q<2gT()WYE-TgOC0p_)cWr;8jWU4E~f_T@4P}GYsrKIX(dmQrZ`M5c+I2P1Hy3zigKBhuB`)V2ApQz&gn+q zCEb)(zmVS*pT#5T^uHU$8ESWuPhVDx-C~M;EcJz?CXe-3=Z@ztH8KNQBJ9qqnMjaP=hNcQd8m(aH_#-~6qOBt| zciu?d9cf2pwm>C>nuHqp+X-b=W2Ni+nK`o1bd6<;+WOX&UMy(wZ}ZR&lc=@NcC z*u%OjsU>KaTS=qZ&i{Q%wO;3rSULHaJ?Zzv{>fUEVHR~{C6mrDtB3P#5)6Lx+>b3< z^zqr&Z#>!_0d_;7AG`$=<>HQHHUIDVasCc@kD46WhPvo0fd< z4hJFk`!sJ~75$e{R?-T0evFV7*MEPO{KAYd3N+conz`EpzRTG?W0kG_^s`>vRl1St z|9fKiZ8dTfDH2=|CQ(ZO(}hYO^6S@EO<3W8Z}JBU#h)_7hsGEf#>iCOzJr0*CIG1@ z*fy5qKn_Fxz&?FJhaiMG3k#WxJMUy=#k}IwPLQbpW~3y{(ogOfN7$f2&-BIeMQPE} zKTJSmw}r5X%)OsKjY1BM+xz%^o638dK0Z9!QR21-jAEjyGchs2-GmN=7Zrm z{!$%)6sL+EzsK1oSztjZX5mUDuAT}{L|q9oG#~&X6>#4Buk&=Yo~n{-H6`R z{2~eF1T4hx8%$8oKOc|UIXJL4FVk_zy#-S6QA&g|2J8;SGSN!_Oq)==OvAl5VFkYY z-;s*V5Trq@BA7OExptCV-BX{%3s`XoG?R}|%XRKQLOuWn2n3qnBA|JgQYhXbO;%B% z<#3x*Aqk}tW^WuEUcVKW#yOWGM15xK$@aS3#9LnEh2N21o8Noij)jBK__ji%ivK@s@88q=mSzRsAh9u7AsKBF#tZQl zV>K6{rGqz(Y64Mc;vp0F{5h)@MFxo)y%WaSCMF%V zH2T0i30g-nPHe~*1`M@JC+S;zo+XF^q<$cPQ9UYTyk8mD9}E~WlMU@*SpWp*tMMSE zv>pQbgEtNk8;E&euUS=f-RYvn1Ij}s4i5TKXi>PX)EIr={pmo~3e6RtnomSZ;3{W1pB(6LwZbz{>7 zkaS+^i>H}h2ql*lbI;_A;`S}DN)13#KcDEAJ}W4&U)ie++72X zsnoYwkP0B5hegqb=g$4@5F^o%pXDPs4$(>@coN(O)_=WVKEHzqo5vt@h{a*^!K^1an0aj>^0y?6 zpqxy?TPEwucy>fXfr!20NG|{u=LIl_!8j~5m!~c+2VmR>`G=`>)an?$eR!bEChrxC z+72*P;^p`G!qB!d!7v9~PY`QGe8x>#c)I+z9I_46XVAC2T9~LuTjl%i-7%uUl8@j= zM5(P!&>Q~7O0f(27NSt=_`G*LDx2BaCZ9&g#J-Kvk*`BQ$rTRaJsO(p^8FaircKx% zlUrVxB*KKnKJ&#lDlNWvO=>}Hv;C!qXgM|`o)5%d&}F_!?yYhl-8j}RZT7h6erL(p z;&`@k1zK+hNRE0=)a?S?a0aarZY0oFZJj zsuF2zHW{Z?D^`d7JxilKt7f7UX16%L_|8HwG6*Bg4*D$FhO!?#0(9eqsq;D_r-;~q zXySrGo{a2C!XWyi5c#$d+O;y*28)J#34}+5LI>gcK12jEVZf~ZCgjRqk{5U9qL-y^ z4JiP2Da>xfq9s-rSNPZDpIC;QfNmPF6WrSVmpzMnA<))^?uDTFUbFS)jha{YG2~`J z66DdNo9Cp0v$CLyDuHHJJ6RqR?saqiCx{9k;6}3MNB$;^lojP?G%%WpviA=DdhYMs zUqiZk0yPt%N=ylfv2tcuE zUz=$q;zK479w>Xr0jznFQxg2I=0VNBa_R~4-a9B#ml1$SuDnIOaXvRfN#n-IH(Iz3Um-&CIu$c`K8 z4@5HuTG&v%ufx{{{E!ZjvR4&BYJ6|ds7qF47KotZ$H4tQLIlD2Nzg+H*a4+&+$}N@ zg`Ntww<<~WL;a-zL#9ufx#tdJM-#OjX%>xz1*o4fY*SC~Wct4v-vrSAEu@A52vV$Q zZD0H%N2ckxgqi>R!#?$E5%njLAVOxra${WT=PBkotz8*k+d6HEXdB+3Da`hUlqItXUJdn-}fzQ4M5jW74=0CG|Q$r1Si z5>S!J$$oMCg`Dluzc1}YeF(X99i055VH^a`07`K-ejw$rG8D7!+_}?rW%!@Ccp;P? zVD&>R6vaF+q!hay_?hEIfx*Po@3Wsz(!mH5-xkBV$7Qud))@3eT_d;llblqIbhL5l zb%6Y2unYtESJ&bXkwG>!eUH`|-2TT|2oqpNzjr|a>HZQ8Ol7MR;7=K(f*)Af zPd`50xx+e!{B6C|ibICd`vv9;c(z~PM)1nbev1N@vsv@L^c@V^X(>`9Fsv^Pr3?|< z0asZO>O#Di6&}rhah+$6U3?8F5oBJE<|+GBaVRM?P(*;8GW7H3j)?@!fY68mXu%y& zjN6KNNW4J09o@k*K4ppvzU9Lu+I&nEVzj7#qOC#P!PHb&7iYWaRc&DDX<%=tOa{tv zO&T$^Gs(BwhyW({`!&_Jmc1*LG9jK36*YQ=1+kv_FVnHEiNUycQK z+D2P@|BzVuA1d#kc*`08BCt@7fX+Q4G{2--tX({bzC=pX-)|q4g3_&!UH>Cqt@QKR zE>kIP00D8QhzL~G799vs#DXr7ayzIXdCP{i7#!MvE(~gHY<%AjHMOp;_juKNuau%G zMf?Jv2jf)*!Gnj}+|Y9(Idhu0c7s!f5h8$Y1SVBkSPMUKl&{b|j{_5Mh7Tg=)-&%Ff7h>ka~60;OaxLdJs2!vrw_qM67<67^h ziJlA8R(dnh`g~CDB{OuA2N2ePwgKh-mx0v(_X?mpOe;>E>P)eFhMCX3V_4>otutm) z^78t3ME~CzGqL2N^pDuzzJIYS4ka1s`+z=eSGcy2V%?O)l7%V|Wp6Hs3rSuR>R>Hs zO6{`wfqJsm>1pF|`&M55+pv-ZK*Hrb;@uu!5qQ{AYgA ze@a;*4zj?TKL=uyA6TQ)?DrTLczi=!$5ZEe>i>c<&mPtQGDAZSCjlE@7V1-ic#az8 zb4)~Gtv*^rS&!M(ECVeG0xL>t^fQ|AQX3KcW6Bq-iOIu?KO~9=Xio*FIw9erM+U_x z6@LU1D}+tNllq066#<#JuZPjrgL{Ui4hTGI&pvXru?WaU#JFHesQ>{ZTqh^chr}SP z?P}I!WrINt<;xeOvY`}*XPu)2i^glx2LTAYw?G?J{1EsyIGi;$1pW!02&3I8Vp2Q1O4Pp6+Hvobx z$S{CkoKQ?bq}l4#X;G45E6y-fieZTbZS!$dyLc{$oh#sr!cxD&4Civcj~Iw>u4cBt zLP^>H4va^H8S~@)S(3p4GraXpvRug3p5mjtA_Upu$0~eobYj!rJ>IDFk$e(GVW@ zm8(||c+H4`#J`O>_w_mueFvTv}@6mJhn$KE&JzYg1-7ih$ z{~vX4{S{ZxWq~Fk!6gX}0TKxA?hxD|cyPDi5ZprW;O;c;(6~bf4~@IK(>OFP@8+8~ zvu4)&3*L|PYO3qjJymti-n({rueEdUIt%SO&)j>D7y&d0Kq|3XjC>doyw-h%fB+;c z1;8ExDAkQkOn`{%>F@W0$zuXQ0U-JTqDz2~knkQO?$AS(l9?H+U0=!7l?ySz=m$MD z85tQlIXMBa^zDi40A8-*5`g_xs*I#0BA`+Om?a=z=>A9G0V(r13?3L52;9dPIbfPw zUXC8#+XbweXK=U~ApOqpz1#2U?R`f?B&()|18kST1_Kz`i+g(V0Sy5t7w8`%K~4@- zJ=CnMu>fuc>_b4*?bZx3QE+kLMM{b}IIwhfb^>tYKlLHtsRi&Xxqy4l)|M&ogea(} zfG!&sodG*>ZYSe(?OcDKnf?dip-iS zPk@e?pwhH_JYLJlL4@(|EiG0GJ{L~zkA1KAdc??7#WP+KZSu5nNN?prk;d~%7oj_@KfciAaiwcsrnH!{!JCj~HdsEwl9qc{&b&9c+Z;W@Dw#D3?B8 zCT9FV;NlS=I*BgS0nbK1tz=m*#FGYeR?VL9W{!+AHePwVjdIzn4u#kgk&^zi)dqHf z|11DwV`J^mjkUFpzz|$2%P_A;Jd2Cn80YN1Wr&3SLw+Zr;qMvq!E7&&MS6bgkaPJJx z;q6@WoNjMw#9C74GTry?$vRb+4|1=qsCk{c~Cb=r$n6IkXY$w2$;f z;vW)}qaESkk9AQ^gGC=bQl!;b$b~}KMzz{go9ycKRAbtK{W&r*VuS?u49mC;O>Y4(~SgG z#ZPo}vTT~6V*jOO|L-0?U;%;k4=x5O*&{b=hO@~@OOBqNp2O?IScG?Yc#_i6p=~6e zdp>;zP7(e0KOuj5lJZ`?{kU<;i{pAGMgE+w_OxhV$R z?rj?SZ$P4CiQpzmX#Z20^{@g$G+;4(2~3qecnzlNzv(4K_P^Tsuf|gH6B+(zK>yQ+ zo@(j;T}k}=8V}WfDP#ZF7yCc|f1CQhj{!9HA^!h>2EZqwCUO6pI{5d4j`IHp#{AF2 z$o!u~6X-$@>;EZl{{P;o{N(>L=-%mF+f_!jyl1cY+EEOI4K9&t4i`3!=f3xDEB{0U zTs#a6D2TvFAua^1i)cSJP68khluSwIF)Mfz$+wUrcbq+Rq1QL5V;?S0NS513sWrke z8dWSYI$qxpj+5qjeCoNr9T#~ISlT+YD2IIJCAyTPKwOqqfVp5Q8`K3U1d)iF4R7Dj z4%`rLZ6`k?Mh`}L7``Z*{UWZ$(+6rAQG+wdA?w6Hw(Oarsoe!Mq zzwo06gC>pg8Vlm|A{Cu1STx-W;T&vWm?dKy_P2xTx#t1h!kBBU2{Y{P*#5g)9LzeEkKNb&&&RTexoVb zuyE!)6iynVuD3i-UC@xvo1c?AwEjn(D#>d240D(fKws z9q6JH3pXlkmZrrGq3C!+vJ`s+>7cLrSa!*1MJ6f7eW<89978GWi76jJ?F($+NqMm| z4KCce_gSpT9;*HCfVH>al;N}%(CFIYSrdF%~@wpMF#Zis-0fQ`Ibdu)A;-M!tn;rx2|}Uh@`rN zKsepZ*9zPmen+hHH3dED9JkwGz7%RPIq9Iq@`2Oe_Md?AEXPEKGUz?l`f8=oFJjqz zvNtuZ|2~AfRK-K%zJGYPnJPpXWUpvO8=ZMonpwW88)dYFjyEDpJ#-CGzQVa$;o|}? zipJANq+aaAO;s$Lpr8Pg49q^jQ0xAXzCoiTd&@W}B|-9e?^< z{W126yBER}TII%dV}?Y=agjoZPgNv%#W%J-Q7|PWxQbWM3u%Qz+xsB^86BAlmLSJt zVMioV*xq7<-hgwlu4At}#n&?ygF9y(AVI zM`E+klzQY5q{i)r?R9P|keVbD`26e|BEfq%f4084 zvjmuy6Tsc}jE`$IV#3FCH3vo*VOp)O2>icG_rvcA!2+3*HYuy?duc{w2YD4GNgEHg zz+tDLqgzT2IG=R;n*9fE+Yf#AYO0k{whXtM%WW+Uuav$~L0V48mRvj0e?`q444D+Q zRwVWV5-!j0>zh#rppI3UvK_n2V+kf6D$FVZ@Q6j>j+@k?a^arw1m zAZw;1|!cse9~qd6E%JMB_p6f*Eu0MFkvp z%YL!v4xT=z|5bALl)%pWOpB`bgVw*5j%{vI(np<^zrn`%Vul1KX_jqGVK<9=GKJt> zPpn|;W&y;8^ZDkH%-v54$#avE(5^Y%;L)3|6VXA-i&pFY6S%E4SbsaftvIafms3kHZl|Nw!$av} z^W%YP*hOOJc+Pr$ZAU1)tRs8C>T|D9uv`X(eW15J{^UaM3>Iix=~{TCZ$ac$-^kiI zXw+<`Cp4B1YWkVxt8%*$?zBdby(t3g#fb6oMn-^koAr_P+90>V&LggwWw&#RuPqN^ z-Z4SDz5bzCq>auaCGp*11lqTs)7yy~-Y-4`kZ^F#sKzx0DWo^XXns{xjrPa3YXQ}j z<6LDy&)LS>BB_r%67jh?O@2c+xUqzW?UfXIcfWG~%_0@@+j(v}X~wOa4@mLqJ@MG9 z9F#^8gmOiwU=p1R?TkVmvCCjb;su}~vLpd$FYmh$N?u{UYx#8G#RWlXCMQ*( zmAR5F6~8~#$3OfgWlASyL@N0Zr&tPHJ^0<#?k>)4#XTNZ2fELN=B^RN96+=2lEC9F3@9a^21XVC=Rb-Wj2 z#|s1aWQ>~^!<;G0Z~5#HEwV103r8~gnzjg3L_h6al&o!=Ho zg$&qq`=Hm2>%-ng7Vy(p9ds4`NH|#%aRFO%NN-eMDOa7l=-=o5EG!jmz@KI{bk2?N z{ibJtML^dwICBO~2dS!PDJXX`9q4-i5dTCU)44jFL|8A|diTJh}- zP+Fk}gmp^Pdo(`mBPLc`Wo`!B;UooW$@t1&XF_|oDNS|JF&h&W1L!pL=?e@509>a% z;;cuKVgN6x-O%(r!egoFqpU{jZy$k_ZjNajRh1RdXoN#b zkAxYxv-bDwh>+JbNQ{&Qz_l~cZ8-#Wq)Wh+~WSlnOh ztdS((dUF}1uYNH$L7KnLSQiylL_BmMwlL}=3f^w1|8Tr5C%6(+uW(4y(K-9j|! zdH16r=Zm#PMtk$UwSLXXl!IKVHR}{lBq>9O83P}Z*FHaQ7B$0|2c?_#u$H`zO2oYu zZ*EeNnx0FrO6KdMWudR;nVX8L8B%eQ8A!<$8V)#gf7GyY{A?TyU>5XRo63WW2M`bCA;1dRJoNN>3ERn3>D zcrBJNtc*~D1%}IDAr4Kt6mJw@at}R(AN+KQNN(GEcA1RrTr)TvLjE{N@e9-urc`D0 zO!;SzOk`$hav;Np+?l<5ziESM=0~98or~K=vb7RbpHkEsFGC$7ITh1JgC9eWNy%pO zs$MV1U{w^fO8|ZK{FQ3>+n)%OL>#O;cArl2iM;2F-(@Spahpsnne+IbmJ4^xzV$ak zGhBSv)sZ>DDJ2%?_ZjStu1BT=ncNp+>5GJMDpXPAL?&o9Y=5M z^+@ZP(8vXPHM}CE>9#UtewuO{E`_c~V7K%KEn2Ymu0#BZ%@4EW%!q(9C&Lj=a(Rt$ z8l60o8y!VWC%!`d?%sz*W*}?0JYyPNeNiTz7qm2mj1m*NtPsmQFgdz|l}i1!^$6Fl zWkY|4i$8j9i%4{Xce~V!*Qz@4oH^NHdKxRvUN|lH)YDkiIfb1fJZ;@ml+B#gV&U~e zR4kIpCiXzNvpv;)Px#!!grKg=D*w04d&04O?@+k`vy}{)y5l`^^9Ay#(Wq6w&-x(>ujgFTAnn(IB8$?Fr0`Y%aD=OrJ3n2M_Uqxx_LyC(7C z-s93-gl??uIHC*5fBV7vY~Q*4?LgCEcDnKWps?kUnPGMh8L$`b{AB~}DDH84c%_)Z zib?jEV^6~&WWsYC*euWVyqG$8{WwwX`?+l*5bdl%EWUz#xI3f}ell5+1 zMd0v6`WK^wH(gZY^MaRU4If^!sfgt}@xKmmP45l1zkIy|N*}Z;8Z{qSzOJ^BBe|j| zxF?K}KA1E7p^f2=!)F3G?@_Mh-5a?XVa3+^TXdt@lLN zW(kb4fNoXaq&G|gs0B}QbHGHGv2@00LJ!DykRXyZ^imbP85ZdAgIB&sY3L*8jkTFd zo=6e*3q)fog-@JN5M1HW{aUL7PVhqP@*7oA6vf}BfKR*1UoYItr|jkxHHTh1_4PV$ zwcS<{dgT`_Uzq=H-6VS*+0wtxQ&9UO4`1p~#Ak2e zik@>lTXj>s^t*kyj77i15fV^jSJZ{`GH zn0f9n4|N&%{TSt=1!v}tuI$8T+U~ut4MoNSiS zeALf^3u|QKVT#Y_p}8UszvQzlg%%Yu|JsaBcTltO+jO~9)&2SlRq+!$UemInC#wqP~~OdHZbD`oj}Q^kt-nfnoXEl0?y(Ilf=Nclq?$RME`1IQGE~ zrabM3BEuV%{#28os(p7BCQaotK@gQ%nI{=AV)*$ZM_OHmj5yJcy5s9^MDjl7wJ1tv z2aU3zKTRm6`9VjG70xOA^AF=XHdb}-GfBjXqphwj>c#V!wxXUJ6F{7p?^5GBdmOma z_5I>U!&^>MKPb9?T|1Ux#PD^TXXt>vE6iMsVsa9g-$aPIy_&X#k^8v_`8wmrygOIT z6w7hIa;rJ&E?By5EU+mh{_JkUl>B(i{BRZLA&%sIUow8z%jWfjlFzIor#bvqYm_; zEN>)ZdHD~Jrb2{n9%8QjIXxo~5nA$cMvIm%HSE^rDM#OVZ=VcENA+Jz6Y`|!Xk9D_ zYiY#))$1QNix8m;2VD+~!N#ql$^~Zk>ImeiI=Gp@tZ!p81|GJCo|`?j!Q6&p&k1GB zxKaMvD_(c3LJgpyh}+Z~vhC(Q7mjL2y-#`?_>!5JnTn0GFX(uf5;r$g3f|wWMEn?6 zZ~JXQ>KD+Xb4+b5KjqsXe9bo26sbw-6_+&xOBM8bCFSi>nZbeY+ZiA21HGhw{&Lsx z>+{bM>I4+k9TpMUfSV{!oxJJ1Su%xv&UF(3)O%e%gALW9_aM)RWbXDSul`x>05TJ* zVZ)*PlZrjUV|tVBJ0~0cG2%;0NP}jp_cc$vrizQyQKYjJRmkWZg4}R>`c{PHK z!s2;_9Bb6jemY^Pwro%)wS66Zqvh;d^D)e#DSr5b`Ct9gxpNoGE{(VPWO3YbXsu&& zlxHAF<%W4tR9Kbgi*N2ly~tgjSGSI1JSplsSL$R|a5|W{#d*0l9l~OEEb2tn_g6QD zM_e8BGG1Ajt-qqfn!$>k%>xxJ?)hHk97FE2o85*os9xLT<3MVyw9xwrr@i!{fcqp? zmbBYlgQ3yF=x!yt>Sw#v5yKBCF)u7#2rgSFY=VN!PXH@dhiqp{?Fu{qo>v4*fu{#h zhbh4JkJyfr4KUTG+p)>=isRlq9I5VV65+ZwYs|IUkofxAgvMm1^?K$(7XIjLLEwp_@_a7%_GoSbrIOqJB&Cu1mYVlj!%Qbz zl^jb(WqC` zqYkU(G*Z!MJ@+;OBbG#>N12D`eaP^;T0|`qbmIvW{PXbmn&_~A(H=j@tI6-2mRRI} z$e%1cgHwnb&knog+%0qvTvLnr7zAcCG$2<>S?+LpKHDq`>bL8Fn}Zo4mhcrSe;Q)U z#;2b3oD6elhi7I;mUnub2K&l$EbROWK5ku4;k=48YG$`}kxaImYDT<&8zs49ePlQc zi@vylSz9$WMg9KSSljrk5#$|OBuw+?FhP_&kS8g`2Gzn1zg^ft-MHhv7_qkygq2RCM|DfxM%xo{%fL=y4_Y(>f8za=(&c%n6r3>%=% z7&+W>K-WIS)TlP@#ap;15ze-Cn`knjNdIk3s3G)OA#31aQ{^nN`QxXampq3?>h=bd zpz?CV!>%CHp4Uh4wB~b9hvtkqM!~5BbwjLC^Q*1mM~zA-tka4(x4tl;Yl4XHj?q%S z^K*WFe4AyndNX0^x(P(tL+c_G4 zn7Fi@9cyhAk7MKD%6Y5p>4Ps%WEyRY!=YV7m$a$*)DqdjEhThIeNi=B-o}%u5JM`I zPJ_ieDcU6_q@*WTIyqF@GAon6Imi*RpG{;j&34J}AK(i_JMluWOd z_CE*Tl1d=S&{^d*id%GcV7oU#E1yj_`s8?aZo* zf@-=%ONscaJtG7ADdTxLLlSNr902Y zI9&)4mRJ(J>iW8v#9}_SYDozyixUGe(~+z!5v-e&1ByzY$p7MZb03GF^X*m6kI-YS zjyi*IS+-7d8?h$TR3)oN0wwdhg+Hg=1cp8z+ExO;%jKzkP+VD2R9}l##PxG-D=c5> zLjZUrypupY8?v4aO`qND+y22eHT^}x1wU4GCkF4#VBu~M*0;2%Xnp;WDcbG2)CQw) z%`$F-{EXG|1y*-^W}VCRrh|tRj5AV0NqUKAL(Hec1uL;XXn0A%#|6VBvf@%J0~ktH zqY2YE3hy&Z^w)0?jmILV-d#PJcevladpaL=D<*RAUlCC<;VkGye5fa?>&W>`-M3*s z;eQTrMGgxgR%J{@^F-2Dj#{roE@fkjQ+06Mt!RKprI{PQzb6QH1 z8b+Azh7{5L-{t2keR4J?|Mb+?en1s%5|akt5^IlGHMv!9ie|AEzfeyX^jgXjaqv;( zjHC2O-JCpT(4G8UJt!{9FHT9@A^o6jK#r}F!p52I$oeV8cZ(Ml3hVn0;Fk@*)9&y9 z+}c~zSXw51=;oX7p&~+T&da?YL~RW(({vUHYfjqF_4CGaC)1Gu(F7ld0|!{3+lQmk z#i+XVlrBgzua^L4crMUEo{v%1!?9KJSVU8dHpo`nd?Za$z6vFJQr2DvgWv5iH^-Vl z)ty|maP*^zE(g`oEHuO=bO$PmDr>z@>h$(4yZ?Tz1sf~Vjj%n7ul<>eG09$jY2DA7 zQuE*?s@(X=6h6*y)_4A^twm-1+kb$*SJxz*x8Qjq1uF-?$nDFINfk^D4E^XGc}E7p zvvl+2wGpe+NSv$;`$TO`F4A{WDtDCPr9;ciRM=9DO-$95i!Cja+0(N{@Jc>7KyLQ< zZgro)YO<8CJ%49}s&h5SEIa&Q!}~W~I;&TNIAuX>I)709-xrr5g}Hs3L{=0;baJ#{ z+d_*!zcf#z#2?%N8$0RWn0@`(&l=CYwy+}8qx6SI|Nd_}8*_?FRTyGG6ka5zcO0|*+yu0_Ne{LJ`C4vE-^@4L{u)SJd<)8gR;;IU~y&WWZ zkNs;YuQw=uVY;Is7YHysAo?}&f(nE_1D?@W(3Q*PQULDaGA&WQU1;}3hbZEn}LApjpzQc}=dpLs0A zPTDSfMoYM5TJRowqtQ*D4r?Y$J4yKZt)I_<577{5osetGK^YCBtI34r*liFy6D<)# zg^l-#6{-870+gT89pTqM)~2or8OY&RsDGvee(t*BL$=Nz(jFF^O2k#x9sjagg#;NR zpfXpWr04Yi2=ci!NdOEkqb_mN%4nI$KAMf6$s@CE)aY1(X-)4do%j%!13$|f_Z^l* zgrlyMji~5z?~I@deYu-0ir{DFOaXx6jhi^_q zLP9V&&`O>J^6YHWMv9M_9yAh+Dhq{Ko{;?6=Y0Di@lXB`0~M9Ty8QxR4F$yfa#M!u zolY5T#Ss;i*Sl0lPhBoF5)8A$Ic0HJcKF&H%GO9)XHU~w09Y=eYtwPviI3HdTSa#~ z)L;VLyig)rK#}2*MD1`v#K?u1$%JNe!Z|fDUbJddnxrxTKe=1hUXJrkPyvU_f>4p#E(`_u}Nj1K9%|{=Eajo`MSsKH9{L!+{>D*GIMO$^g&Rh<1Pa z0=~U{xxe{0Q|O&RQDVmk;%`^HugeP|=NO+nCe#Yels+zq(cZ^TYF`#H1VBwI$b zcg(#xbNik&UEb3b(Zr

OiuYO&ogIXf=v1%V51UdKkg=gTBr{lj9snI|_XNssAp#`l1}l;B*!RNM7@1-shdM2Ry=c`VoL zDx*aF4UvO3l2s|-18qdPtic$&Ig!dK7}6vCd_p4SrRy5(39Jqb`K2Y;;c{|cO<%>= z)}+R~jfxhDRxFZeEqXa+`E;?0G~K|@a^0Y!@lD*t840h^|8MpFP)6ESz@kdyASYY0 zs(vshNBaE!cq@4FALF(Iti5*B!_yVtmMDy{b`6md%?g@YF}Z^>*HT1Ph>$JA>hlSv=ZCF8xjZYPlm$8eV0;x@BRd+tqq^d% zt}X=4ulMc|OHZZXu+^{;fZqCp22PjZVJZ~+H9C9W?eeP&NNNYaxO>0fjDL@ecKNu% z*q%5m$c9+TKn^bc)j-M)<<3AMkJkpk6VBqm`vM=~?ZJ_WO&kLw01>E`lyqx*DOnCYYJejdponMB% zl)4*t(YFzM{Ncu>+F&9`FQhZjr#H>AV?#P3n_px~IVyBPUUqK1Upse3gCPkqe9yk? zIB##a78mb~d@-FTvhKP-uy>SOO%AFzs5G2js7P>;aGadPde5Od08LIlp9mqgJg8rD z@ESi*cr8)nFcCECu$*WSPe}G~BzV%U=$S5iv(eso>#iWgh@hdX$lty5DVPD%J6Dw# zo?F(u69;1DwHv(_M8~=uUz`^`yW;e2@TN>hT@1~1=EPDW6x*gAJza|jS68&4QA<)~ zdDGi7Hv8un|2!dyNI~+nv@bUtOk>t+{!VhctvA$WMl0iwC#%uc!_Mp@F5}Fs5kg!r zy@`)AJQdHh0XS$F`br9Pw2+Onv>aV>;Z;8z&AR*9(qft*pPj{8mbY`vOpni4Q?OXk z?LIiM?t(l&48*?7yJekA8>NpH_$x;rzE|Dyl_oe1)#L1>@jMKI$MKX8a-em4URUWm zkfQUEv!JA3_08heP{(HB%9N2YavzrPu0GUC^CTfKp>nz%_S=~72t#^{ez{SFK0doB z;WO{5;q$~>#*NFe7e7KG=ltv?^1dXyeKIK!T?XiRh@D_Ts+&E%C=$A z;Ac*Ic(-|Tdc2WHQ0F#r^&Gzsbh^`HK5vv`JA2f=oP}|yCO5|6XL$u|Rx>->T~|A%QaTMEGH4jbXWMBx)|S6G|;9>2^)Oc(Ey@e$A62F?x`l zjRh#xmNt>3>-g79raaU7LF(TMGg=~S@K~K?E;*mojbHV_8YXoN{z%zFdVdd7{=n3B zw?7!wZ380O!YU~_Xv61g%^kZ^cO51ozP$QD zkxApFL;@3j48;Y3KZ5PvLiTVJZ+at_9D7AY6M}Zq*c$A0yAL>t=d`rLZ67?^IGEma z8{%6Fdk5jK=!=+HlZoN+%btj-K^=9SXUUrHrqI2Y1Ey>Y2I!gFM-JNP`0hih7W6C? z{38YhGj)!e!!OLK&VBli+$)w3wQq$+h3<*0(Euu!&w(C;YW3s$iQV%-1HIASB6*7M zATZ3U>c@1g$F^77?F!vNh(b?q|GVPai4`>^F*#;Bn%FlMdR=DtS&~HkXqPsxR_CDihk8Hlw#FazpogZ3V===Q$8QlRX~3+pwILpd`S3o4+HI# z|L@O&Sx$NDURNQ{3-u&>``6FX?~O$ruSJG$k-7M&k290>O5byjNeE+US!x22-Qvvl;VKy|Oz3B~^x1+{+1$vv znf=-R4BrDbb!-|=*nxfiM|`9w8s5bKY zG>mYzba~GH+0yIp{f$|c@4nb?q866LCCkVTaV4H8dV5HSwPLzxsEpT`Pi1F=(~NIR zgxO%D(V3E|llJ?UE53BD+sa?3CBF5A>3VI#GuWwJIB3>LzHi0vCZAsFl$F4UwFD1b z8fqT)Au2a%SVVv9CsH5aR~H1@&*)e}_wTkz@B~m3>O&Y$1V<5?IvFO>XZEb4_zPGe0 zCiffk1P)zvH*|exC?tF^n8E3~OTW3?E@(UL7IO& z`9BLQnxp-rPXR^A6?qf^}0kwa4M3 zx5mSG2PEqZ9oJe;SJtl@u<5c6V^w`t+bkNU`$ufko+cWt=XcDI8$|2h zM1(9@o973o(I5J0lb4}-v!L##kIb3Vtv8pE`0lU^TMkz^t7-MdI01^x-;Lb#r#}^t z6uAV@u?|{0gCN{S_eto)we~IH;%xL zHz9K`33|L%?2uvT*cV|gg^!y>S$`8D%}MOhAo;(?5_+VqsVm1!MD%Vop4+?S+90Vi z#)v^O=;s;Tew|UlZeoyqUUUDi$xR81Q%VM=NLAlk`pVWrbSlYP=Tos)%neUmuD~Y4MvNC zDE8937 zEPHShU2~>MXEvGi6nWDaX1?svDZ|XkT+UcuSBu_U_cwu8mdnQbthD>Q*2^y_u3s7=CnlE z!Zd3=hb7MkiB1RghBgP{9XV}wkNlXo;kX_@(7!o&N{+M+3Eiq3H(ziMDWDMh^py`U z-po>9U0mU}5-dI9?R=FI8eR6{CnMmy^oO+T3OyQO99wqB<1#q@no-`V|wVb+(lueqP!>I*6MZU4n z{j(81YIM+ay8Or-)V(Az-iO=veC$=eq$b}#+fl|&aEA7q-0O)~*LgWHBOzY?kK2dB z9Y?MG;)OnbX2_pb`bOo|@#y9KuA?^22XK++r*l*Kc!9m~?Af4s(Djaa|8Oh{MU3wiCzA1Q88!;P^TBNfcJTO5_E$}N62d$1CxPc zubJxnR!6mK=QKYgg~|I&9r`v_PhoP3M!Iv2{p$b=vZOWKw>{{7)Q$0@**i=ow3Cd! z0>g!)9-f1yWVGI#?6luZ9h>Uza8aS3oSD-p)S~@;K6e*zf}u|}&V5)DJf631{;)zRvX5lZ4M${a0#2t;dPo3QLl;>pi523%gd zs-!}~m^CX+kCH@#`?7i`v5MXkf4_5mF>{m+E(wiJZtt}qM&18Vwc)T(orXk6aP^jr zDs%GRn$)uPnM(ZbC^MvW7uxad>J`{eE6h)YFTOuGZ%D%Xb_8O1=xX>o^G^ZyQ)(~T zdo0WKwnZbOY;f{bFdJ8;b|-Y2xa5G_@s30;H4BfG4L?uv*8zrlfsIyU8hFOeSknXh z(}n$3iQ{Iqn(YqUkI`(t^c5dcu%id(&Ffy4qsy|U-KeruUap%zcOMU8&PsDi;RiDk zj5x!k9!s#VE= z(Iw;0F_6_!259Lx85Op(JcB%ao?KI3GZkukr zX}xxpWlyN-c9B>PhGx;yRaI6K<{MtQxs+bt`vp zg=SuYEF3U#FHy4T>jBbMoi5W|5&N;Ii+ivxXI^4Rf@&MVcfZ1|7Ju&r$>*b6QN=F^ zroo`$@WkadtG=u2b+U{zJyHcPv&h{NbD(||`ZPKHtfLA3kVW=H46K~+F8ji=Y}({v zCAa#r@xyb>;v+JFJiC?~oWv}GN^f+&f@vak-I<^R=IGS|#mv}}nxbJf=iOn|+CsD9 zKnK5q$?P#A^{IU9DFl0aRUg-A6=(@x*=$+=Ns}%kl(0zjK|!`?nkLe(u;mGST?QKg z6|vPiO_lTZmw7mwmktb!LzTelfV~wpk4UEPplWIMBk}FFV{s*LpJSFmM)A(`gDiZF z9#oc_CONI9Q`q6Ti{jE7Mo9NJTfO5`*ukYvmRjoqL+eGEa)tZxbBXn+UUr`p5^cOl z5y!$u&d0;nY1kmC8h#dsrdd&xvIte?E48^mG&Oa--D{!I?o9QA%zit~T}G&K^`|Fj z_9Pw0(?JjMQ{i#zu>dC{yWI9jS{`uPHP4BPy6mfP`>V)_+t+1>llJOlxD6+p9xi1^ zO8Pl*qLHYVbekKHvInkfAyTXK(&d}ryJ=dj??X+xF4K?9Wts40s7*D^g(0YPozFsQpxYea#b<`&veXt*(a|7(K?SUg^{Xgr0dO>$yx?5z-lVogKs~ zl_l(cL9MOTvAtb0I5@T*)nr%MOn?kz*J)?|PI0L5>@c(h8B|}{sWy^4UF6C}7@lD09Hh`D&s$wabJ3RKK=I|!LRp9aPN z7>AZ4v(olj6`w0Gq@-*#tEf-n4Q-x&5l4$5=Y5xO>Fml)I^Ri8Go7>XFI0|w97{Fv z1gk!SMbO3s9HHlGUrS+9#1bX%YZemvL41=HqrMq?%54%DCx63DJC9C z11q+yA;PCx4N4|}aQ&u}tI>9c<+AR$nF!J^^6sK44u85SEj_F*K>+9Km9jtEy4<$O z(Akbq9u#13es`EFwBbr8I;hkH3Et{(I@=V>3SEy>@{3NtieAX{H|8`TcB*#WF1+%m+^p046X{z%lDpAp7?KZsw0fa2nmTt z_P+I1bF1|zfAs{bh=TpUAfx1mT&^ez@7_H+lD~Z)vj!WwuRQV6)hee9$+&Mm(W~@1 z@;R|(O)s_u+~^@ux$)LKWZw2@Xs~(?;qK=W?}&q+8Zw;ul>*@|A*eBRyI z?0k|_iL^-FZq=>^3uIuT06-S&vy_zNjdY z63g&lh;qqcgASIiJ-4vXm6m()aCX!`&;K_SHIc$ZifNdektH{

mF%!`gwhOR8j83mcWD;LRuuCuD z!$FV?`uXKrjIh07c&E3$^vqP(5DCa><|p!NiJoPBLOr1?uFFie9%~J8B#%dDAM3Gk=#J4IEV3kP^9$KQx`{Wiigy!5~2x zx74?{WzmiYq2itkT5y@%-@DF(kafU5F(hCz*r{uZFoShl=<4L1yjQcdNWvtw6_p>K zfy|_!v7*AQRmbz5!yR7>i{l-=K#Xc9USyKCEpeICmE(R7&jOt-0pFfcqbmKrg*xt&Yb6=82o*eF~ffp7n*1lJdn5wiMze83^?T=s-4pb*`(SW*fq~tCb$Y zTNkhIPQz;3W=uQ$ZA+OiLNs@s4HsyH9q1jQxK1ObB-#Inh=dJAjZ={A zrm@c!vujDGc^qvaGI^N`NnJ)}=ZVUSyxm`8|MmRsUeG|BB`N?hdLx05ofqC=B##Ww zS0Voz!_Q)Lu0q=E+I>P)KRm?mm*C~+86YOANBf4R<$6o?#Cy~L^TQfo-{T$l~5H9y!4(rR#V)B?5; z3e#_vTF>o2B;$Ok7nJ~!nlvBY|Inx-s=SY12w!YE+<)^S^}QH=eZsGgW2`i(a>53K zvv`VIF-8=;jtlaN={#{VM*3>9Z$#^i4sY+y~3j*pfW z^_5wrk@qQl`{5vxn#XF$GAf-X088*3c~^n7>nk_0>}!8n@QU|szmGt**Bek4XA;Fp zdS3{j8w@8i_)AN1Kd~w!Ba$#7=!p&ya6Q7it^ca&m1+QC2{X#7j38(WO=lmRn<&iD zSnON)wdsAI(-(F|nh^H!Bim_5vd4xj=OV|Q;?;2R>?w4i;PBnrKz<4M++7A6j!0o$ zdwmf<3)<=N7u;LVuE~NdfNGx`!Z>FYO9oLa+m@u$w5_Ys4zk1ILFD=}WvC)ko> z!%z5dg^jhX=|GUd&2juDA@-hPy=CO|Q5>oH-E^8Ov7xC|pQ7(66vWfuhrYbf{2?`f zY}ea;12M#iwYr`OM2$td=I`Tm(>)l!pOf_N?H;e;C(ZIjD|j*zCd=a(%KrnFKxw~T zF22gMhtqlBzyWUm_zA{%+EIW&+lb+ObNQy5{OeI(Dyh4{8<@4sKlD8x9hb_X)1Ty5 z8`GIxP{N{|bKLsPGEV3fNVQZ-d#@Ni@b@*$Z#0s$P*b1Hwf|bmStEPV$JvH5sYpTM z!EsOji{E_m0>`BGVN{SO)lz{FWDM`Om<>&BR@u%==RY1D$NocS@QtrF@^p3~ON(-N zWa~G4Vtg`PR8lJJy*P0Gn><`#wVf7yGcWJ`nv3U8WlE$EZRG8-+t7#k*%zPi;R(Yy zc*HnPpE;8gN2f8u%brqEpeT3as25f;zd?6h$Ba#7Ja^(CfBf(TCc9aGkCWdZzW(#} z>v8bO>@pgt0ti7u=itG7XV(?# z?#bj?sHx88`*V6w(ayv8_j`nUi(790FANPlykarKY(xq&JEs0=1xs3uznhmbYwLJy z_1`(Br&XUtiJT5@J`4$uWOP&{!-50o;bcpxL_(R$oA7bHPlt^jo z?8%_82u4RmF+4PgUak&QNJUDO4osT;KCd+Ee%&0Fp8c4q&ME=|{f9rsw_jYq-XWfJ zcJrivfIodaU1%e>az&z4#eiPZ`R$RCuDMrJGc&h+#OZwksa8lSQP|ScKa?@?JsB4j zOjkQ4B~lfA6Yk^tTh6oDVCmM`m!96pHx|!gzZieoDP?h{c>pJo2kCU1*iAagUogSuLAeTiINFg{xkk z!2o+1Woj2Dj(?6jvTA4~sc)@h`p^I3;<;lPAK*&0LP}8(=sMt0{*qnaiHES5^ejEI zkxxw?#9@;k;YUC3<;APTEGfw5o*&=i?6ff2$fQ&_23vi0jiv*p(>E1z)hp8(;v2<9 z|J=m<>Za>lS^kOjTs$_8uAu|?&X(NnVL=*Lcw!UZ{mUYb>=Qv38wI7JNQG-CA6tIz zy8Kym4b0r}IiHv}nZ3h2sFqpnTG+T}`Q>NtbHRN>IAqj#&X_fW6GjbSn1>xDqCknl zljC3hnI%p7ZgDezT#^J*dd7_8TdVhS-@bhve)jI(!OiPd@sl^6A!}uAm^K_Z6Lr__q{t<_UI$MbWbs(p| z_%44~vyPi~9^}=kb~4MNpohcJQKt}IA4{+W23K}dXY8uO#opF*oR=mJo-u4s$3Nj~##`ov!XWrybyHD_H zNjb}kb9rdvmwaSQPdZ3JK%mk&f{Q*mze4NW>^`@g- zpwz~L!=Lz!7prs~bf@x{4>>T@o&p4VBuwP$jMmNu(qd%E$!|C&$d2}j^SC{;vV(EA z7`4pZ^)3hY8O(odyu@yCT80MZT{z1VNB8l|muE6iZFS=Ri6qId`;@tS5d)kQT_#C> zeYBN1e1CQ{#oHuFjE%)y_0lLh${aZ1&7G`j?XW}C)?VhPkEC(H+;^E-(Q${g4Vipt zSU`J{B!VxjKf|V*K4R2W^U|r4JhE#g9~l`%l>kCiGbm;-XFmBHKV7|(>AQa9D-Ta# zVstFWJoYgw>)L2;$l+&y8cJ8WK($vaUs_wxF(J@)m0vtLjLs5~w!Xdi#?XUcdp|7tO6cxl_;+1FD^K%m;A zCzr0hKutF@*%oaHe|%{YeH|T{_S8?Ty9G!Y8?W&5#UmIy=?z|~ZtY-?TdtkvYZD_W z7eG;N%U%zB%7T`zeT2U03jZ}Lk)Z?Tal^&-M3BWyqh3p`ritf&e1?&(YOCLM31t5< z^ZD8r>zPqdO|8*FnhmVU+RnwpLn#9&Q@JvE%rs7&@hHF8c7|0g?I6UgWzn&3Ikkru z<)T29{{X(T^)hvLNRmig{TY`uf{(uZAvYg5$*k;bW*ptk<$rse1HxRaK2?y>C2}-B z*a63~^La5LFBt@)$Q>tm@=j?wlks zQBz;YGslke$etB^ASHk*2?*gfNfL{RRTsDM>2Yy%3rOLstB$j=!xx%ZowJh*60Ag# zMCHukFRW#?{_ZS|*`#N6VHQsx+sQW{986cG)yCMm`Z0Or9KQPD4@}Rhvqt2>d1catPB*5~r8XU5Un){Y@*4{<|g6>p7$3~d$M`LBhH zbe2;P0feYvKJMOp?u)&jl6a%)-oGddwB<+8Wl;EAW-2D z#F@+1vaq>xAg($6HHSsIQG`Id=y6AMjvyIdw)ejhZ^1TD^>e!<;EXpwcs*OMblANnW)D8#uG4k2Oi+ zo5l}!WYciV{*~oq@c8kgT=&5x~u6FK9uio&0@ae$w%W%aqWoRi{b-3x^sLwk?m6R&>E zou_iy(4G+2*XME7l5uob%LqZi;NF8cV)P6?`~F&Hmp5B&n8n1_^3#0hp;S6cL5bRx zBmTUeH8;zzrIqJ5{EaC-Dhdh*_Ic<_mUl?1>Ud$(TkPvCqo{P|peKK5|9cCKx_ahc z$l#IfUvPH6pmy?3L63lVj-CHBKV7|>nOBNgQBlIo!&~_F6ZbLDLD8NlabW+4K4wn+ zJrq%xntAHSS2!Tpl^&79`Tnje)R=GOS<1Qft(o+*mESf=VllMw!j^YAEGCSjpZ${A zH8&?NEYvp@bJd@R(@7#wkjNM?U=ercHPS*0TdRwim2r$~KY5^oe)?-Z|=l(W3(G7qf%mk7MIKDFr$wNob+Lprf{>mdC$+oWb_i z0v4HDDEkcdhAFo6ETO%NN+&TS);T?R``DUV1q@@zxeCFRXc$DFF^t zdnfRvRTn$hepAy`e*V}Hx+q0LkkP;IRQ_Z6&s?`*3-_JQ@6s0DQQVsH`Ta{%8R+7~ zm{Cvhm%OG<_NMjPDK4AZlQuH3%Opv4;U%6rag;m1e~Tl-?I-}9lO!F!RL3i;o}{mx zoPfZ9@ymFj?Dny0V?!1-77KN)<;*;GjOklG;nZY5$^{Uj1LOLR;Qi0M!f!Y2bTxUX%iq+WB+#-!`&L zV<9bOn)F&~T5GuLty%PUu*#}~PY9C+P2scet>*dCMw&?K>I=E$!$%n9V)YAlA>;YO zxw4KJ%%W%B;jcN|M?yeCx3DQ(Q`G!BGX|ZLBzp=gx&Pb0FfGKHq99NdMG68a%I(-= z$}2ot(#CtqlUNK*JhS>m#@LGl1ghMk`1B7KXt)gz*;<*&x2H!@3Q+0Zhi~q>(kAj*7o-~I>v)CQKZTxh(qT*%a6WW%ia4A@#v95{AK4B{`<@KxOnauMh8c8*0S|1 zH{6iFtaWd2Kx`NXPkn@c|McJ7b?6i?UCC!*UN(>J{DFU-(T{Fw2}QX*6Q?iZsY-oU z&XOg^R$Bd}07}%J>^J#QzWwDY?%0>kBS#K%|DJ7J{mlnlvSB1q8FK3aiGq+HBYz1fa_N7!5 zsd5YBgvVd!_Z#=}WX5@(I<%ADy!!%2_3)uWrex6IKXK>fTHbxS%EIQ#LS8s~m_NMx zIOCkH=Py$^GA4B#7cTi1H>aOv{VIF<#i(V@?sqxBM{UKK zZgG5J)mgUvij6ErmY({SV}tEz*K-cHo~!80mF@W_2lW}mH`ix%k1L~*m3bF<^2AYY zS^f&sg4#KVTaqN#xr){*ZhmDVz3i;T*-inG>@|2AU-)1hv&x%kqJ^5;t6cl`e1eeNLKd{}F8TCb{vHXqauv&U+-!vn1+BP&)SE9bn2(@3w4q)ynt8 zbRHs(u3v%uc?Ea8f_v@QR;)jqgEor|`~!pF=6Z7vHVHg?j=}i8D{-Q@0d4vw=+vPY zGI~6`ZoYpfh>*xskjsAK9*!V|Oe*Q3*R>ROEJ)((GtzH!@+5}!@auTRPgKG;x)0L&gd_dxS!fzcaA@mhWKQUZ zRBu;ACA$J9Ka?Nef{njqL!&pK)g&Nx@B?^YTpa9WL}Fq;Oem~|(lZDiE;hfpRWh{$ zf}#@O>+g-6*blKOvl0fQ1r9Oe@cc_pVQ`Gw^;Se)UcLwp_J-2ZhPS>ygnDfQGJaW) z-IGQlWJCyL-MX%O4LetSf}hg!APE|W8RG{c(DBwQgA&;LC1Bn|b8sXh5BqYfab(?c ze4o?~i53s5@w zBX!I;oZUxQEZuuQslpA>z1#qk4~h?M!j=O$&>Hk;Hc5~$^a0Er69YRbk&xIAlZtAg z@(PBlqY9>yYUCFbp}|1N%w}jcZLk2>Wl~Vs!qvqYDjCt(T!qV*@}M6R*zMtcW9I8d?bK}rA#B8_XwVv+#8P9 zUs?v2&|Vllu0J*$%)>Qp1B$NHW7g+yV&33TT)zb`D&e0z3ZwdbjeR+lXsjy1r7QVp zGDX7uHljwo3jOCV#?tvC;cNTrhawT%GaezHc3AT2M>t)kMMJ>}tod;>1}82?w1Ws5 z8j-ngHNN>d9kmt@%$YhDX`!84(MW8(F>vPNn0nwOKG<^=`A65_LS$lv%iwwx=0R*&0UXD`9dZ$HEj`^ymh!0VVkJOTC{dX7op=n;W| zLz1vL>j0WSwAR-_f0ueBO5|`0jDcHVIGDW~|K4&KwK_ATP9d1G=+Ah3Z1jz5?w+3T z4Ge^v%7o{Zeu2zdJ!*3HC0ILo_XI4y0su=_`Rvo0OAQWwk^GZllQ-Ek{~ zlBwO1I`v`9IerR%TYCYe8Qby6SA7uw?jl4wbg#vO@!DmaFRX%AuZFfw1EU2U+PA5^ zlQ4B$3QCIz^F3HgLtQ5R{rNZ8o>>R?UL!DZ%piE#-Lghe2ItT;%zN;DoV@sVoG35B z&hI`(kDhUO?!E~8kqt-9+H1I!TMn(!0>Na2R;PmzfbzN>w}Y356QmLW`sy;|6qTaI z90R2k5T!CGq*6Hh_dtxF3R|xLW|-jK>wYYGb`E+6JGNUJ8wB+ki76BJ;+In;xTbGI z{^`>wH4Z^cH`|xdtVRB%T-54}fMA49ql59L1q*@*Cr?kV%&y(YC^(Adh8pB#WueX-3$Jdd zm>??P9gzTU0IfDP*!0s+I9FbA+gPY3V>>?kW&^SsmFPEq97d)_!1k7Qf}+e85rb#q z!C5D8>FrIZEIx&AKl%ziV&BG?P+JHx1=KPH0)oQf=V1>Nw!tE`M^xHOy!7-$g#Oyr zRN;i!Au};GZ9R?`)uFk$1Q)W4(5UGHd%L@LvZ7Q0wG>bW1R=!74GN(Y8bO7?q%nAW z@k5AkR$ljEa>oD+o;ns$E7MVOwm@+s9 zz5VU)z72<2i-OZz@WnSU$5MuGITtC4=O5^Y`JZkbF0nN)(np3(4CNKptqV6jEtdC%af8N=Xj-)`gV zyf9$Qbd24z4d*sohPJr`XU`VEoZ1n)S(>h5`<6Y(E^C20+!aB=LAbd~qr^T4qs9)w zciT>*zO)t1S^+^lQ!y#otNXJ;%`LceU=6-rwFk9=FDA@dfC0gs&rIzRi-{9c@!g)w zC@|_faUyPk(Qpd{^xkru*zgVZUTOKSg#G5`G8|a@EjFL5!+@7(Vo;cKM{FpuMO4ZF z^om%8ixv53sknsQJN9Doz$ipJdm%o>3oy7MW5Z9_cfJ$`vj(+BI}DjH53>dYLn)*p zE-?-FSG2<3FBEF21{nv|;>Rr~p&sxGrVk23$Bvp3xJ9R6P_GF5l2wWNmU0~2y#p5} zrNJxO84|e?D!CFdQK7I`D}W{wj8YGbo4**3O-+GEd#`_qlRr`?&cn!cn~`(468fr( zIFs9eNip7#{Rcarn>AH9w*F%r`0^_pFKUKkP%n%d)&qs<>v1%<0hjmvfX`OO!S9*L z@U!WjGzv0X1Sbx|kfa}wbD#{m#&R6nxeHe(#3I`E)}bFJ)Lh<=9Y>2{5~K)D8I9qw zUcWiR@VDCh)fb+{%1=K>dQLSYjzPHZ(O2;LlamqZZr5R+9BIP%@e?rl{AJKB2F}tO z=j|Rn3`^ckK~QWW!hM`N{JX&5P=xs?QFHBitU6YVoZ~yN>*y5pm>i9pBS?LHCVu?* zQ*1tU4ODtz)WRiLx_AzHhd6h%hcx59iIXw){ArXali<`Pt|zqhwQ%ioKVEw4HB3$p zXrC=^uSy=iv_*OEorsw*p!9 zT8IjJ#E*CwuP+`2FZ*9-OnAcEKL`O{w$Ltq6LRWDfYkuB=Ndv+V?`Z?U+>B#u zzs9$lPebDvi7}HVA}PS}7C$0D;Shk)a~{Tt6X)>7{vzZa{28CEOhVMFGvRyJ{R%}m zc!eU-D-_0hFO(#Ig`ZAW-iA9ObX7T6|JBFXa;6$R7B0mDqvG3JWZzhqQ&evZOp3(u zyyIxH7}3()0<8&!AZR%pRD4jv(Z*2Rc@Yreyg!ji5)v6ZWkD-a(Y0$ZgVXwkwT^~RJ( zp20)odtYzGr*a9x;K_3_bi+2}oveknG83n-He+nKJEW4kJ;^_iNO$mx#OT=%W6`2T z9ex%*^dRQVo{mw2lHlWX`(fWQnG9-`;?6w`sX=x|CdwPyKnRdXMTocjLYc}7{Zf0t zQ7#3{&B(h_j24Zd3zfuO^h*>v{L$r`%ha${{n{7P6@5kyL60Ek_TWhk@6cEzC&t3Z zK@Lkx4z_IAgPeK`oIJcCt1Cu9K|zO~;?f$}cz8l3kpLiCiq7EFl}4D%_^&;pOa=eM zQCRr&i+KL=xk!ulg7kmGK>53mJ|UH>AeUN6FFW@r^iAp9OJ1UON3Tf>Ff}C*N~r)E zYH(uz5nO9*yKe9VsS?U|OCu<}(6et31iC73;nfuIP8g0^Q%51lNeWo>D9z2l>GOFo zfM}`8#m;qGP^fW&zn20HRVAI|ptMm7TPHWzNv(a!%L*W`kik`d|W(ybv{f+&URS3MLI2md}jh`TaolgR0%$ka% zAV-L>prs@i$4_UX#?ZZ^X~^1vpSGVxok0S>*hED7d7{hQrJ#Uc$_R|=6An8Wpsmiv zwyk@RTdwI=B7?NRVlu%DKwEGWKX2NDOQnr4UMDF9sGNMzZ}?z@I>=$UCxMeduCTq{ z^Y}k2>v!`J0fKP*;ie>EGMNC278iE>gzYD@QLnivnIx(Z7}FoA$-%G|4&IAVkwdLk zTAi1mf?s5B3`&cFoy;n~c5Z>_*((_lzRr+{0;JBqNa)=cu^~Z->Ddo`;%*5B1rees zLJ$C&3}|Vnzl8|7E9pR6$szo__AqLsj_`I=qPe{27TqjFRlOEYULKGO)_$KAmyaRy zS~msXVu9IWhS{h^{*iU~`M_l~+`Pw-!8fTth9!n}>MtwE>=BnT3d8z_LID#RN;0uy z%YKyUcuPOlHk|)uKMLuFenSVsy9=-?DwR;SJ5kZz3lSmSu)QV95acSj`UJpLEdmgs zviCz&ydOHX36?m)$J-flnFt!2P+n1kHtp>XLFyBSk^OtZ%U0Tva8P@o|AbkXHX;%B zQUMl26S7Vn#F^|WSOBzD7vh(V8<172hukp={rUw%)uoLSVB-;m36loGODTaxUyqC} zKVje1y7qZ4-Bc|>1sj_?4CAF4Rj1Zt<)+hUvJ1kXw7v*%?u=<<4!#&Z_Zd7fe=;Uc zUx3FS7zuYpcZp&VN`(Rvi3mUfXWwun#&_-mEVJ=I%Gf!WIwTwl0WfKsaB=@H$ZF~y zpfJ{4#*VE!kzL*fwUak`BqzW@+V#5BUXd6)b|gX_Brxk5abf>j>^N1~eOqca!$J$p zEtNR7Yb|zXlt5#&-0;Ps8sURRA|=rE?hIeYH$uUo?bvoK8?6=_1V;2gM36(5_*9Up z5tA|)gOWm_gc;2x7qD~l4qR!n{Gp~?R0W`KS}#O~1|zmtZ}g1tza{A_Kq9sm@t8Ge zY;1+z*fF6kh$^Vs^Em)#2i?GpI1% zz{3)$0!oF{D*tL%#HFOd@76*li4y(^_u;{r1L3BY0W_lQbUKa~)mmpy-Q}ZiuE3EE zt8wyL8${a>r1T4e-E9V&DV#zvcFJ%B*eL*03$pjD$BxWW=*-s*H(Y{JFnwYlILQT= z+RAZY-9}ujX}cw6Frh5#EY9TCBYOBK^l(#l-fE>1YNg^g6J4TGd3_d$%rOYP`^LlD zz9YU-*gL`3&l_q1Fq!nIt*k=FQY}JTdKDM07Nf;P2!aTSq!ZYJM^az(^mT?zAX>^x zQCw9E{XJk_U3EV8Zrp&3YZ}=2CS&NpkS^z$2%;2@K3-!bPEG&-AOJ~3K~!-3jpVsh zDuY_B#D9k(Frg|x16#K2M=d?z?@3ga7IuoS%XK20ot&T)0hn5FCHo?ZE1GVcfkdT~!5Dz=(tMagS%Vu1vCAc zL2dm)ROep8rIHHV4B4%C&bn(Vk5KeVh=F^DVaOtsHl7Fxbb$cCVroQHl?E362Mh6N zL`C)?eDUt5IF{cC`@r6q|H2!1>+gTXGxNqE$VmocQ!%!GzZ`4Rv(aYm?%G5+_(vga z@Bnx!MbOxSyo~)ge6F%nA&l8jhf_Ot;6jZKqS7BjN2eml)#i7}9Su#$KC%g$4;4TQ z1wseT#nX>YMudl5C(cU{q_Fo4Me2yr7|C$D?nL5gG{{9VEcPJaJ78(3fCgSmhv(P)(xib+hNMPgahr~hmVN@?a z$Zlh#q)HDAp7{Vq_4LP$9I*(6gFgljPD7Zl0|c7T(pZU__PL_g$AF>}d$Ik{c{EvU z5I1BNo_cgL!rg2-#6l9K6XFI=#{3y+a5a|U=%KTyY1VeH(6rs($gN}@6t?b&@0$W| z`&+P(M2(d;g$6Z7aMA&+SVdTt( z809ayg_I;n3k(hUSo8i;yuF3DV;}(GH_RI~XlQDM!9;)tv^2FqrzdWKVtkhBCoy@#=1&eDJVpPF#_7cV>p~q z2{RQq^xZqi-)z$b@n(UxsSdV2{!mx|iH#JR<_4JH)n2_SSl1y~g8`L|6YTA!|1%pU z+;E^$skCcCU68>ic>wxH`QvzA16qx>xO$}!wT)(YyWed-S%>VSN043B3=357@bZAY z-R($DMH8C9Sb_xh`XY?646A5PVo-2igyG+CWZ&VbOts}x?qF}eb5;fU7@Wu-o6hFKW z)5fGC+TRTdi2%FczPR7d2tj)9B}{_ws|-t^wsV5L?d|1Pv=79zXWl{lxH5>Yfe7)v zQ5|N|Hlj5D0xsp3LvIEk5@w?jrfwr-qU=;UP8K)7tkGiI^1mZvT^EUanxSoKfv2B8 zED&I8YeHj<4v4b7`%u8i6A>Z7aFXSqI%hxL`SUXnac7 z77|nFI_W2qt00%Pn+HJxiP&|3qf#iWL=p{XYt_MMFvj ztlV)1O;vg{mK7qWpaj}JZm26ikE5rGVIW}Z>We^U`E8+tC{x3~*I@MUT!o_iHni1T zz`^4sn4RDWX}1S`-zm8W(9(<(d-fr#vIVxGJ_z;q>s0;wtL^lPABCkK4ua7lK(16m zCft((UEf(zBJF}mX$RBp!AKd@AHLhNkgqYIu_6mOg*v3U-R;dMGph41GkKJvpn;@}6SadR?gB)?`Tpe1f3UKCRCYpx&!KoWtF7*jVj{rBM=houNfq&zr z=bMnX@Bxe;+#8|Z4v-2WTq8!~u>cE{-7Oz9RN?H&vnZ))1LU4?@$`oCZLggPP^20SNK?^)@kK)HdN--X&z@ z*FbL~05gmRGrEGhLWiV^*tLGkl(2H-gvF$T!FUfz5{VLV6JNkLF%!@zx*#~j55GbZ zi&2Z3Yk4?%E+5T$>kUn#$pDMxkHhAsx=b8AaTU#a4GNBLz_KTfc7YQJv#||2XCL@G z5Q4%1`ub|Oyk`s`+}0Mii*LmhoXE&Vt;P(w zlPx^GyrI0kLc%997CplKaUka+w9OUBJaZbw&1r~p6mQ6gaKo-hM6uJ#1qe{7l zm+H09H{>Aw=v6$D76Mg!;iA;m9eu`4$H4VRvFUOXDlYEA`h9bdJU8KbVFh(HIDhI0 zicFDserR;pNtv5`?cMrh2qGk1EnXslOewc&jfF7k^|%!#X#6&!^b5@^mg*b-=}~AK=LoUB&Z+zNHQd7hm{WzYxuB8W=3NUTh@Xy3bwML{XGL zt+c)mX@O3s2YK&)ru&Zty0&^$X+$^%_r*g?UdJ=D(%@ku#qhb0;r?^k_+ZOngRZFxr}yncL8}>3yI>5RG66k2ZST0dqA2ZTgGH(A&Uzwsfv=Yf zlyVVl(V(ub9&I;snZ|M)+IJj9wQUg8fk+!U8c7kZcicxoltF%nFO@sTAhmxO>^cP- zA{?Au;Ao?Uhz6KU2IzG+Zi-P~gL9`(BCo6!GP?jo_Us9p=2GN!by19Z*!oAq-*F4B zW@VzFxeej&>TV$f;ij-u5JbpilG`MG9MNO=2*iH52m8xRFf?63W^N5}u;S4hK zu0c;J>>NDc?ykOlydrZ&LP8<}JvX5wzXi2LmvQbwE+!=PgY7+t)16{OQ%gNA9r*

0Az6Ybcag*pSz2A2lc?U;zCqZH$qT3BXQ(h zJUleuH}^u2$l)I54Y>sW|I^*b>mD9as(?>IB7)UxP_8k;)LMgz1_LbJ#vuZ1Ln096 z>yF(QD$!=tpsuz7ZCWE5a8Bt$bi#Ck`fa)%}w`ycvDn^Ue2naTi%4D|d`;k_6?a}MsGISDD@9#BZ+Q2c*X z=VT6^NSibV!w=?QYi0$?GI!&>CFihj_jt^{|9(s!*%!W!3drP&|A^(hPMkYE~$U~q)}U67e} z2ntbk?`_q-7&>J-h8;eS4QI>7>P^X6jGi2ew2Q9+@&4UsKzK<_c* zF>uXp6dx->_2pyOzW)RU$KKao6>UcO*`qj9szdZcV-V*g|DC5J+#Qg)9vxa>GMRDD z9H^i`SbQu3+#GPZsv4G-MpV?+q0MMOQ$Z$jYmC-a$>dPnF>;W?&N&=WAr9D*-v*PR z4OgrqqF0>b|8d-Lzh3q$>K)jrmx29mri2e*r}K^X9h<1i-W6L3YF>}&Z@B-x%nk% z(h*RJkji9tz8x!ZLU>p(oa|&M*BDS&UV@UcN|<^&{edmF1Ssvj5SQ$A{r?zSD{=AE zF&sN}9)*>SD9f*cwOEs|-U9qR8h#_dy4T+$QQ9LoF8Rhj8(UCt@idMcJ%!6fm1r!> zhsJ#UYyQwjQ*afzHBB(d>=89!0v?&x`?rH$Q3eOENXW(i0W|f8y&q9uR*Hga70^?F zS`ZEOmIbY$4Q0i7D6i5Z&QW!bQo{EC{&-A(H%D>9HxaZLVKUz8 z25jITl?eY|KV@t~Y3_L(IdTM-3aU_5l!ZEt<;KDNju(?Tx*{wl5-!pVl$lJ>)K#Lk zjez>DKr!lS3UM*36h;V;Ik>~i;od@bb6Y;LuGGM2mLo7G4RaSxLV(S0DgacDKJdMl zti2$Ct!GapgxX?PaWgFDR#a8g!)(6GM4H8nwxSETSZ+-Y2vRu|67ddlEkbD*jQ9v= zh?#XTn{~*~%0`oM7@U9Y?jXqF6_twn#`eXb{N1Q)D8Vo5H(>UdIP`Xt0U%l{i*fR3 z2JCt-Lf;_wdn;J@J!rR+u@#`QwSkRN2?2zrp%z621<(z4>u{4$kl4e?!CGA_b##WC zoh`b}5t}rqE;x%z*Ytp(grmC`9Hb)tuRa!?230v(xTYiIK2ezXz(Yv%cKFT2j-YTx zjK4!?S-UB5+$H|C=ulskgZxq*M7a<8PJ95<620$8R{x4`{bKx~6z)NCg#UK&od}1( z-dOa~GCVUQ^?Exn8}DQ+c=8csTwaRpmuiuB;0JuPG6A7Wroi8>J19v4pM+ElNeaV> z{L^UCR^WL0A>_;$29J>T;v8ym>6hI&SyT(N%o)jPgAf<$@jL5?Eof^gMCOH3m;l(g z$04Ps7ZiU`+Z36s{bnL9Fc=ISy2V&(@^CS)46O!0<>igom>9VJ&IFLm-5(*&%C7yX zL;~rrg%Q^I)i?Y|V-51ITt%&(FrywPH~b4v?!CQ$8{6tps^$M<@4Vxys_*~*Ja^@e z?7awqu=n0mP*4N}MNyn}+uBw;UDeiVt)sTJb=Iw@I1rS*k-b+4D`fATn>)`v=l%QR zMo1t5f>5pB-+4Uvg9qGmKI1)K?}fj-klZ>i+QrK6Hkb%c?oXeNc2*a;&;w3NOKQ-x zI*uPdMp>mBAq1n*g6WJx-9SRe4ASDGFl9SY>nb^xlS{RK1fdra;y~~;RB>?cK^k<3 zii{!NYHD4-8BAF1=Ly@3mm>#{Qtk2)65Ewauf3bgdPJN9mJv$(F7^{xS$vO4+%#(u zS+6dmxGtX+@BWoN->hQV>^a;pdj`Y1Bw{uFu&^@aJIF`V_bpiCyYyq?ga9dykn;m{ z6$ibB4yHrwYDydGDA>1+Z3lDdpVqyJ@gVprtEdkM=BQ4L7~Gl2Q1kbHbK<3@rjiDC z0E{8TW(;EL^b!33*asL4O&oMN4+mX>vk?qNV~g1rZCnf?ktrlb8le&p6~F&HTMF1? zNs0@@VpMQxI%+^g^#`adEyttjAU4_$9L*J%_Qd`_S#)40DJhl^vk}k;s4AMKanA81 zy^gACH);To0A8<;K;X>3wj?K$U=3#0>za$&dM^P@+K|ksV2w&=#?PO?Eg}5<{bd}l z@=$(cGw=TG2wPSyXU2`UbIY|8=n!SYp!{ECNGt6J&*af3-58(xJ4<%v;A|*k+t=@q zy>~OKR?g(M+izj=;B;)p|E@zNnxDGLJa(*F!s0bMsI#@B|Dd7Fnv%#tdo{avAI5dQ z0Eg65R#A^Xpd*%Wx(yu5)Cp-9sZXVultI&Y=+8cke}0bnTeESwE7-OCBaZFg!OCUR zxb@DTaQToDbcq=NX)fz!zj~n+HOZpY8RtHImu^1>u(>1(x zH8kiDf1w@px-v>?8VHD>>-t5_l5eZ&7ZDLdRHO|h&yVOje!qI|a$5W}G*sbq1@BeU zJa{~3-kUKZg1DF{tbktvcwA209-%Emf)t~jUYFm&ADsc7dH!|QAFae)na#=%|H1yv zYgsmRHh126JwrPsU~QuzrMq0zI-SAxB)W#*=S4l^ffz|hh{IvCqTomOx~X@&QH6^R zs_B}ahKd5VuUpLGRokhtq%v^WP_DVWGbN5}R_)Bi72pT+h_2ytR*YlJjEU`g{3fd@-zm0wS{Ge>&e-6!Y-IH8OSkl-HiJC6&qvj< zW0VDgNT$(j`u?gwF2EB|{Zy1z;1jSK!s#|@B3JfKzIcfY0ToweDK&nc3_IpOzNQN+<(t5jy?Jx4%e5l_M?CC&VU~LZe}0a z(tTP9@5JQEqj`VDe(K9T6z(vve# zT}^p?1FAq+XgG0k@!uIMG}!Efnv894OzU=0U0sV`1o68m%-TuuSzltqW>IlD16;g9 zCj*v{NRksHFi?Sr;P?3u-9v3{9d54xI;!f$ca}bcrr1~#!ox5tpo$NtvjLx9;DWYH z7ku6dib|Y8?>G=ZYs|;E;Kw8NlvmW@S9Qc3Mwk9$m^wMVjkK&_jcm_VKYs$3gz(~f zOM_V1(I8f~dG&H;%(KZD^-2659 z*;(w_e+R#P@OFlGjQT#i+dxj;j1j7QdfQ=H5MI zJ@8xZylNP+HsgQ6tr7yBI^W!(By`?Oe_mYiBd0!ze6;huO645b+})WN2Ch|BGtI5E@E^ z)zUcM%Z$w$avw*Wt6+rGaBM1lbO)aljYV%v*n`9)DoW$UqC|y+|GJx>+8`8 z{Xv|)#S)xx0AMtQ;0U$1foGdmXzKVpHB?o3Ia6x15@N++vj)#ug6(NcZ5E^o=CBmT z&v}sew9dTo@@p(uzn9_~4<$#o@#$;F*tcg7_y6v<%pTjV)!y}I-WQ1KqrSR|dKD5w z&Y zTxn7{ta?4vR8^w_t&^`6huwxL$dDLK#>RO@|2H1dd{pM_x8HRg zqk4BFCd@+d`h8rm<4{b5gxE2g6~KTwBoxQFWZxXjhDcg@>N#Bi=TL05oS6<@G+a<{h=0f6r(nEG#%X zM+FqK1=FblMzN4Kcsx_a_GMendTPp!uyXMduD+xzy=*mPeY1-4gaM51*R8Pw#Sg&t z3Gc#;TW(?3kzy8X%Vqbc;C}}epEo$>M1IyB%tYF`jN_3 z#Y}AbK1`fEf`ri1+W?#Fag3UI2e-}sn~2lJ?NRdR6iqDHLjc7FTdoea;2`rn#W-4a&t{UdK7 zL_~eJ`Xx(bM`r%~aSXNyUin}V`@epfM{#oN%<;4fRXMa_A@kSOGW_}n_~p-UV_@nJ z1EfGPW3}2an-#?EqWI{3j@8d(V5H%!jHSBWXcimRAEDwkC?;a!q8kSg=^!qg-$tL$ z@d*U{)YrKPjflW*`|bndblrENCwE9<2Qs2<=LwYi@il_321g{RDM>_x7;rf37!(Cj zOWD6C7w6Q$M4qjrUJu~)s~FAJRtIx^4<;!-pBfyDWN?t2nnFZ)8zPd1NaA8cgYEPD z0ZI$ebKiN7`OCf`GpCID(!(r% z`vp7(J1;(V8$IKHjM>R57eMve?ltgq5zt4vfVsEoSXfQiSA25jrp60$0mCvyp7h0FNR%LXvQPxH?F)8hcya)iH!R`poSW$$A zvOTkVw>dl}joZtzj$?QFZ_ksLsQP# z$Tft95*-nSMFEfQrlO>PimCuzl1*oK7E!Q8Mw1W~by2HG#9zaf&tBqBe|?>;)v?_2 z*bDsT`q3muH1;Id)E^eRil4kKpYhBepW)+eH4MA;4?O&v>*=1`rW+x^y4&?-o%7H$pk90SoGPKR3?n%{#$>>9n<>}b{+_oq9}xf z1i`_8Xw(%KQl)An{IB{f1|xQd4Ml-~H=jMbi|~% z`8$d@w&_b=f4e_j9=eIHG3SoY*y09q<)r?6zO#@rx0}p$tJqh3CGBh)yS9A8q5L{b z;T;$>@e2CHUI2JgQ83x!hz>VFWh02~@t-kB+tQ;pl(Xrxm-zGF-eqfc6^cEEcIlny zlhKj(sVO8SB@!1KLDA|r`S<4=X*MIh@i9k45aqC8Qo!SGpr)o89mn?~pDcCkC zC_qunqz#{;5(pAVDT2z3N!pb*}v2R#zw+Lq}r8Z|k)*m<-PLBVKokUsDV ze)HFV^V&;KaqoJD$`;f)Ea`9+CHm;L1&g}3vRFk!PCq)esRFE<0Tzd0w z`PZwj^8BOsGNxNRW<{s==vv-+e+dP(o{QU?TvUqpt>B+8yvq8lN+LRsW!B7*Bsk0$ zw!~8jBRMGsi`jsviyMdOyJyaP1g( z!>{<;%P;fSNABg4-YHnnsLkKW$8UW`R$a@Q!f19n>A`#R!swCy|yAirENyZ4T=;Y^Sj1%fQzHs*K?rQa}iHc!&zTL zUA+g47GgULq*uE(1kMyI$*G-b-#Dz@@2uw7!Glztp<|MG+&CLt1O&|C(RAw3n}jy{ zv0CL}*6+-r!Ka}ZjD#ikX4cRC$ZN0vgU5b4i$19l7y@qcb}r-NPghawJ9qvwM5fR+ zBaMg<1;kHr{!wxZs?NacLVbNbUcU;8ndJ7JNpBy25!HP}A!_A5-u~N*yuUh&KwNKTOudY5$>HBqc9}wWd_2)MD+&QhvUak0@8Og4 zP+F4}o(i@sp2wQZs{g{gjq~4Ibh~zBq$XieKzw!N9z8^1&6ya|O`Wp=RTUIV6dk&B zr(N9tjb^{T>RcAQ@dEEG-igzc!i4K@U~s$eiMH3s75Ik0xL+N7Hw3 z4?mVwP=z@5L7&%avB&ee<9UK7pmF{92skoDg}I?N8bj+wzO})A#3)QlX2eWnD;m&$bQ0G@%8;wL|E^hdi?xHNl+|-z=e?!-sP~=wZjF{u`c1xy zfr)lNaOG@d$(CHI^0L{yaSJ7Gh3NJ}xO`H-3)HUMh$EsieY(eB0Q_YK*m|hyWM#=Y z8Lb|UZ~2U;|NIhbGD`{TGJ&5x{sM2k{}FTFdy{`Y{}d1X>VEFHDCS_+A?m)Tma4|6csh0MOiYMQef1HxZ_7tB8Zn;3pNrdqNl^Vx$}8#+ zz?RyVL7gKo8NaHf-|vV_A&vvF!?^2eRjxr6gltZ~V7@7$Tl3vM1MgxF$9 zO^yuuQSLm}EZ$6o=gb$YxysqKZ991tt~SW!P?8d32(cIueN{-HlVk|2pr zLs=FJ=YB$N?BzW1!ryuRxo3Ii=_h&Oi9hn#L%-tYX`@MtIO{0vlZk<%p|vCkbw55; zZ%&wUR`{`10i4pY6YUdYFrrhNw~J3dnnz}dw{?TlQ_i}N-r$pMWmv7>uR|lE6Hryu zz=c)miqS+=x8V$IEEU!QF0!_6q1bbd#){^n{LlvWR7KFccMp=jWB)SIR4NYcCEKZD z2+d&Pgn`7JSKvkP=4W%bxDu}tN=DD&4C$MM$!H>`dq28_TL7Wv@CyDjcQqvq{#M`P zsmf)=y!Tmgpb823;q>JaPagXZ7vuIRBy{S{$iaPxXbYJt7P^cXPw&J?j0$KK2U)-E zD0Qv&yBBJYWmDw#A(m(cjvPVfw3rrwDqYvnv=$_Gz(?_puX%6b4(j!jvcPPQWx!=O z^82Tr=NB^u5Na`?R%VlVy!=G#gZ~L3ubdoo8JrIzEu4*pzg5Rm@YWWy^7HrEe60TD zuN5=V9R@P%p5OD-<3DF)nj;vz$jqd`MG=*P&0q4)f=!e+1X?vJT$P7d`0ksm z&h`@81~H-6=CS0{&)Iyul0Xv`9hKIXtM7f7XCD6r1uS^=-+Zt-pI&1|(lI&We>Ag0aOUQcU+YA-#?o)ZLRJKs^?--$^&j!hr<jj^6~c)G5ika_Yw^K1E@b*$W# zjXNMXqGE`TiD_QhBOrQnd-}|4)=4%@O zJ*S%1u2}OsIk5B{-uYq|fl%v(mzz!3tZM{LIg7SD=J>9R9NLF?dobB1=iqKKkCwH% zH}RH}S6GhArx2Ey&d^aK&RQY)<9Gs86=$+~Gn-_fC;02>yl&wk}^mwp%|phqKAw4YrQ->z#zv zY(RWvto`Iemh8@NHR(q*AI1BY@cz4BQY_);(Y{9TR3BvD(K>X+LC6;XD?m<$; zKxk|)CXDTZ&8VPjK5{m%A+yHY3ZDo*Z#BoVi_uXC<&NgW9z^f6W}$kt^oRpl7Td zMXzVqSMTx9H|dYZ$;bb|6Hh$O zukXE)vAt5yowiy!@(>$w-TILcZUoVBmK|Wl%1k`Ew3Xb>3l+KG{*Tp8QqMkgXbf5u z?)Z{7=WnC7HQYjU>I?Vt+1np*pv;Y8Hn#PH#3uEmOS%n3G-?ZWF!#+bI8@uZJ*w6h zu=dlpS$?1l6muJaxwg&v6A2LbKEQQT1Ounv#XZ*yC(35PQ@WQA|NRQfGD|tv@k|&@ zM5PaB>Lo$jQT0}^WyLCX9@@s1o%saJVf30fossPw7YxLd=h~(XU(jc5X(iTl z#*7P{OXAyU4F;ohRKzMHJT5;WyYw z?>3O31Jk+i{TCE$VbSy+(3=PYX#NHcEPI!a)*Z#y3LWUaS~9;`$R~@oQLVP6ZKZ|I zeR>feYDGb%Xxmr3JAXYT&epLEclA-ee)m5tKkOph3IzO=?p(yX3%66(3@eMF|D`wb zyQiM#fvX#_vZ`z{v&-?Q|EuBkZxNUaslauOKogWnL`VBBQKTjVyF~=OmCR6oO*UWt z`x#zep3i_QW-zF8GSSg7#3v*Y9}`Zhm0LBgKMpTFa@zt<({(tz#Tl2{*tZCMpP;?b%c{Z!w ze~}kIKT7W@Q|J*Ja$)YRsUvWX^elRSs+_$nc>i_Y{CF`33hU4=6fY+@N+Vk_mUi4G$! z=gQ$dh-l4Q5Is0FTv}RM1N*mZ=3ss~A&Fg>IQ=Sm$C$uiCOT~x6NaZ@Q9yN$t9$c=Ik#|iCpsIo3)SJ3;4s72;M}o4)33jb zj#e`mE!b_Q^XgR{ybt*n%4(I~+Tuc-Y932VMqYRrR+AA0p|+%e!m3kvRr64ie~^90 zDuTZk!RPhT6s{3HK*_NK?A&*VQ+;!jBaVUNrZa712aJHp?jXc?GBAS@f;Get1R=yt zZH@Qj-t_=pUz1jP&1m)vr{MEYTUXlMyb51HPJm|3kKBmG zY{6hufB;^P6Sqr0@%w=4N7W#9;6%oDi^ptG2-N4W@U<6s^Xn~?)ViCzPta5kRr$O5 z;Q6QcWS@~srcWfHEufPgp#Inn*6z)tp_$j9V6sNjZ^GqVI=nk!1~8g!SkKVM%M_Z( zs4K2zeBXEsnwR3N4IK2hW8Clo#Gh?Y_BjGW&E$);9{6f1E53_761A3HTeg#1>Lj7t z2(FoR1??S%prIMpflIEMLEm^QXg-P#tYqniyq29W@l$qe6FUyqV-8E_iW_D!FeSVN z0JQo3TI0Zz`x7+1`8g;C2Yn}B$-vYojEdkY-pQNK{)P8f?4qpJP2gMEE;?RkB?nf0 zz;myxz|?Od{Sv;Le%E!a)j{#zszP>c*g&4!Oxlp?Tzgq}>@8(cpW(txFxrtHL*kum@SqdfZjN4x4sFp2K+u$ z9VFnTZ2vkoWS^=J3|J%EG47ga4DS$+#bCr7Vr^sFMKft5uVBWQUPM|9==C|QUa^j& zrS2AwL38G^V|OMM^^n|s1k zcUO^DQj1^H0UfPz9!_g^^_et_u50Mu$$-$QEH0+T6?C56b=8ztH@`Psby1R6ic<{& z3R)n5eu~|qC|0bEtx-f=)YLQq5_I*1{!mONEat|UFaZ^}v;IWpyXZQq-;WaBlS?P` zBidmG-OZ8JAMpIYK4Rzbay+WuMBiz+>q^+R;1&Mz>T2x6u4Zh9u$E;*>qaG-cIKdm zVzFQ~HtI&Nn|im0Z+%M90{GRFpn{z051q=}<~T-OIhBz;lCc<|{@5;-uirsgiwPz= z&YVN+K6so4C4!!#rZa6!hh`vvh@6wa)-;U^5=%JuY2GLK=v$tWGrvwwu~Xg7B8tnr z!6EnlYKjY*+eCW6Lq&NRbuNG7SqFl4{rT{VXq4w=vT@5neAYOIO}~!Iy0oQ@lYz*j z3`UL}O030zt~$wDKaWqA9H2qfPrFxdT`4sUjdvnGT&|P$mC+Q*;7cdaG17*DPSv3m z{Og66S-CeCr?&}!B|3h0Ek`$h!L!f2!SRR@OdXxp3P==`GcCJgE{?8S!S-S=_S6B) zm^y~E76QAi-@Sq*EP+uoXEUN*CL>7vrv6oPXvaajZlf60 zKM|86c&d-_#mmq0`hp#lI=#(!R`|kO~B#$({UCwr|Mq=A_ zW8|m-gqsvJPc>VUZlZQiVKjEcUK4M={{Yjsui7xRr)R3R~4Rb$RNkO&iJK|D0S}OrgQJoxKw}M^e z0m9NpaLtv&Ns0(+Elbj1W=_9^4zX4g-9_exHEiCWON)g}nwzY>d&n-T!V=es@zZB9 zB>DXPG0{)T4mkxdHrk`=ee$m_(vA*oIWCFDG-`CbBBk({S6^4*i%m`!X_YL9EPSGo6=ClBQxzf(fi= z!xqkjQ)p+Bk7s!+v|xM)5LH|Em+=07ldR3~f~AjBm;7j-UQV=yR~X|JDt$frG?sw_Tu>gW7<-cG#c zXhz-mNB;fio5%>iFtJv1crs(JxrHmgImD;i@~O?=&YLg1K!W`NrVi~wl>O8JVS?(Z zqabT5AN~7z-rp@u`svL~=y?v#i0HVhve~;gpCMzqG@GR$s*}A--{<4CIau2d=gxa> zVQ{DDR^KW9at`dsq-aEUI!A|`6c7@i`sgM;nzxwat{BGLb{Drz>)n`1tKf*uVDh!s z@YS}zvAxhmUEWq+ed0lK58lMg2?I!t3B{;+sVX?g$_1aY@}SOT_ddpzLy}MSKo<>F zZ4BCpj_UW|^$8cWNxJwc*uR6L#lwm16ou*J2Ju!O<%_xV*m>MdmoY!#&fBJva$2ge zj>qFe6TK0LRvQhxlLTiDP2#d!@8jBidAzgc5EXkD^WK+}>2~*IQk%{<5kKYom-5-l zeRv#Qn052r%pBRVNvkadM@$Scp>_<^;I1qp^WZ^hhQ<>XsAB)Fy|}_VGo(ix=x&bh z*~hANSJV5dJ~*0Wo+$<+Ar3nhgFI&XRqZ96KtqxK1;Po{+ zxw?waueUseT2%agUt^Fq*z-D|LG$!C1xH3Ny2jW!QtrZ;wVHS5enFCT3K?-W)P`zu z_O55q*ULHTvlF6#UsK83zl8(YsbnO@6BQmDZWgG@X3ery%$U%NK`Bk!%_$g6Rvh*a z48~A;jTl0QaQlgWHy9&GN{qm$DEPG+4sPE=(X{@A$3W?!E$paDVBD~7L|OvDtd~am z_4_=*fo3h8wb_g>*PCiZ34^_R2y;M*Lm z@KAYp9e;c10gmmT!_?9JNQ|_C>Za`YUY2||j}0XeT>I+>nA|xOM>FH{F?@6I3yOReI`kgN;67)c(I8$Q z{%-*|U8m(@91T_Vq6S-y69~A20j9Xq?E7} zYjzLbvMiQ;@&yOIVGN#i4|h!)OtjT-inF8R_xes4j+)=!Vr2v3rNQM7>UUH;ZZBG( z39~8&qPtDv=IfTQ|L+UQtMgH?`)mI6ivo6Bb1j#T=uJX|1+}4yLtB^e`I6nlPq>c< zCiQG4gWCRR)RyFsc_^EaXDVI*cgbOP?k*s4#6&tEc&ak_`ir$((c?PO zn^gxZn5;HJ%qC2s@$~3FkhI9QaiAr#2iM(wH#-iOuyAKShu3_;$IHjke)b6BY^EmQ zpNHd{SF-qSpmO~oZV}f zzhEgzGlvsn@>5fu&#v`LSh~9qpQeELs3yrL3IZ3DVBo|HH{qPuW=z6G{}NgbyL zQdGR220V@WO4I!KeQjO2LTE%dHj4=bKMgs%m_L63JrZZoCB}rSri>%I*0W&g9-Lkc z5E?3qC@ila)m($pU?Y2+=}a8fiBM}e zaWRpEn84|+C-?Avj+IU(I^Iaxk-h9M3eaoBPzFuAfy=($#@wy>_#28?@#d3M7anHL z%uDInJ{GG%qoKN(U8}$3vt|29oNym^%o@%)9OWC*oFr}CEAuYs1&hG8x8B!VAzboMu1;5XS*XKVGO!B(j zG}J94Vw`!~dF#bzh&KP6X`}m)5@iR?OGWN}7QFK@8w-Md zxE?5H@7fLQNYqG5Oe8grP&VoS03ZNKL_t&~j?^L3nLTk4$LH;!POWF};@A0gMLsvq znZeK=?FbJs;A^Pj=#G`l`)no7c9VGM_G?HFHMQvXtG+r8@7Yc9s2EZsEX{&Z<%d@D z&fJw$7?YTM)1Ay5-IG(dE}93gUq1mBp4zEFG1}-fYzBA!WCwYF|D5dFe3s6ApUK18 zGqFqasuZ;@pOqhf#`bC>Jtp1EJ+mh^_ZeD{QMQ;4OrCum3sxUwU4a){!T=^u>WAHM zHs9HY+ZVJ|&;oe9o)(ibG?yEX)(Dbm8i5u-8u9yaHMoMm7oC9LciOx3+Jk)f+8cbe z;UKZT6@2iaN=8aNk)bxsCvXlECbI>5XgJX^aYRRko=1hOLU_BLOuc#rEBCz0zEU?i zTfg9Me@(=A|Ba07)q!Zck$|U;<9k>0?#F8=YV7h5C_BvBja!NHcP240h2;1!j6fiu z;%-j?!@D*~RuSAD4}nJ68VthfEd;4m>Yc8}GZK70?djzV9Y7b6;dzVKv(pzQG#<`||j$ zV~9Q%_{iuWqu(Wr8}c6em*r4hu!Du4hfuAX=|1{$F6owdVdugrCgOXH<+i(SA*=Xh zw&fjS?(+{(TYQw+lZVniE)27x0wLB zY$#>v8_$#Ba5Ax92cqm|G{2jQf-F{k`VQ}}*h{@Cv<8@lCJ*MepZau+HZCb(nFPCuCdMRnc29p}?IM1qF!RM;x z(B8e24IM*5sJWTVTD)&5?|!tFI%^u!@4TBSLpqRJGo z+ufMZs^f9FP6j>z3|2byzm#idEakwfODS=A$l35Q5C7~qTd%o>ONVzS)?q<&m9uC4 z624lUMcU=}F@0Qj9L=Ktf~&!OqENx-_256n;Izf|;p*$JVCDWd*jMJJ`q&2k`RIcj z&z!@wvHeMovVrQRJa<3KKKq3A1rBCB_#{($Ht}tA4>|kxuyWni^q$d|(9^Io2Nr_? zN4q`@>6VP6?d7Maem@>>qrT8IJdL)l@f2EcxjiQWAiCCyd<^(;I$bB}$LDPfuAJk# zg^2>9gha~F zn_rLv6&`8+CQbId_ki?Fu!++6T}3g9)e$Z+aS4)~nkF4GI!m`6y=1_UQF8gr8|Bx} z{72RwDieR8`I~fKg{=DbPo-N-h$tY2kZ|cdXrf#>M-|;yF9()BClfnHiva}1A;WHcRyOCeUBj zX#0p0dq6RW#Stkf?K?<%yJU$9wMkfFce(E2w`5;&y=WKY+8X5Oihs!Dp7CM=L9t8U zS&zv#dCpeX+PGpVJ@lzu-7&0jttLt8d4qge&>$D|@z=?ouV0dDhqM>F*&vG1D#<;^ z$sNCcM&9}KD_OmMqio%_MZQ_HT)zD1O?l>_d*!l$ouz&EQF8AypUBY~we@%FPTBqS zpQL}7BA|%L7Arl*UMo-jXQAvmQYdvUw^S7!lI8FIS*{$=UXr_xk>9+sOiEm*pS-(j zo&0oIJ25K=2CF1w43b+O`iCsqxL-tlf?I&BL7-(OkDo&MlGtlO5VF~h&arOfI%V>JISTDJ}e(D-Xz(jwc>HrOWyu< z^4ep!NYCUr={5OYd4I$4(|KMGh_iT?JazpLiLeCEJ-YvO^8SW=@t(?ys#|h4eJFQN z>Mapgi$t~`E)Tx+rDT=VOF$P1c>@Y(`BTZM@wf2=b@4k3WZ{cSL?rp_%<8%z&U+r2Vj)WZvF#IrX}I4F&SSuP>Jbn@IpM zJL0AP_^EQkO>^X`Nu#9S;7jEfProKF{cegxC_&j{jqV_q-uxSRXZc>K@#<2Ww_AR5 zNt~D?JIf7!ct`dZJ5SKBuDRsk@>gYMpCoBN>_(ZpIaj<*)}Z;EvgM6CrG1!LKoNU< z54rlD-^$~U{z`6|eT)2c-X^Jbd!+K{O8NcO10=kWeiQpnmseLG7q`C!VWfNH=<0vV z}r^tvdP>(D{clH(-QW|O$CzBX7%|g4z>u zURPambj^R{nnCR(ZP2yy%Gw-J^|RAm<+e9vN{={k#C4UQzP3f&XZwx6UbemYGwB*_ z6$Qi?)lVM&WV_U!LR9|x9Qov_>m=ENfI-5N2goDy_KCCEqI7Xr9FV_WKS;tG`8{#u z-LfdNvdP+YsXM+@?it%f>?TDF_Eed4=bvQ$+FepmQ7=B9PwL8ZWaC$F$^A1&Nw>Ze zRj$F|C=zrR+xr*)L^ zw>%*$50;7V)HBfoQnGikJTNIkj35}T((#gC%9`WV5@^n>Qh9WV+&iv|SPj8*XgBsA zS-7{Nx$M&eQeU`DUbu6jB!`+#&<~11EVeL-j)|6Vdx(UDr^tjmo|Vlx)o0Patx}Ne z`{4J|JHaBLh{YZyJ;u$JCtsg0JC77fU4vU{iVn$=xBnzp4M>y3E@R|(Z>*H!djIM2 zs-Z-d{`;5GE5;~*ghXe^o&Q)Xm7c&U|5X=f!4CQ7FK0-n7@L?JsWR!#r)Bx>Tyd$o zh^|U??hbkP$=ju0dYX*7{t;Qcw^&YBNvh<1^Oj8Q9w7!0lRZWTPrXwfd;C$k@0Qte z=Ob@QR+V4e+y5gIx+O?N${=~*wdImudr~e2+?BHN?cd1Y4oT8~+5@umaIKurF$R3~ za$wOvFFh&~(}MSIbfn1WX|v@gcibzleYhd0KRng4=I=L1lFcY61_@2-BiH}@ zVR`(KU&@Ww-!3mM*eMOFCZfA#``Zsl&$tj#P{bOWAyaOBP#%Bu*K*e_H_4-~E|XHX zE}E}i_RfDwCiO}P+H4@8h}jk<3GLFQLwc&jMmWSCnI@C&epbH8u4=}Qnp!X0KmVf) ziZux+5)#o~Zu{59Q*^m1wZ~V;15ylV%9X!U~n>4m93vwk&rU&?lTBsUC86*Gup ziIxfX{!0$k`kU=f*W7Yw`7<)Q(cd##qUEyRd?3fGn%hSKPlc>{<)r+wCG?cvzPCpj z0>O2=O83cM=JXSXF?jzbTcq?FJyqt+xn8DSHcl?NVvaod=SOAgpj0t}7;RB9VB)oM z>&>^w!*8ya25&$lkSnkM>@tZpD}rJYM@$Eqc++p>t*5l;&&waGl8qlcA>%qli`fwLyJ9-@ml?PHLjLsB)AFZBeKH8LX%6;lmb$GqpJ0d#hCmXGj(0Q2L{@5$Bd|Q^3*1E)1QzCm;eS8fgRIj$W=d=&$s1?=NymPTeeT0y{Vtr&2|}f z&s&n$aAx~W4|t?-(>rp_;FLzc&msM0J}m3<>zm06-CHH=U%o^BAA4sVA6Jz=?&prr z#3t^ZCQVXTV1d#tUTj%h7uiL27nTK<1(xr+{gyRkaVaisDFtfO-6ihQ_+&E4OvdG& z-yf4gC8aGbr7idQE1xEvx%Zs+yn4=iawD`uc66M0^n)#8z!%!bm+#&&U)*(e#*n{Z z5{2hJAXe-i+{aFYI@gIuE*mXO>MstE3$@NDqGRJla!Q)W%+3`Bg(JnNu@l6ZGiHh# z@BX!T@uQ8R%jx@CcFwYX#nZnp zF1>2H$d1%~;S17+U6fhe2jp_> zZQI0?H%%6%A-ObVog@D7@ix)#4IUs5-ho!}-eXsajOg&#s>~(gr%!z*+6Vo^yss?W zojb%c_g^H^qjX>9id0$=8J8@w^YTPiT7rl$7)9!+3&azzt`HWlyw7}hmCq4(o>M4{ zL-AJbv^&KITRMgBAkLcX?i353xlc?;j}mI>OP^0487I=S^F(e|nlMKgM0DyXaq|Te_s0AvWNEvsPcy)c7@EvYM?duVr{QVY@9ijR%9*Zgx56)UE z`h8zhzaGl_8f+5_pLEdxiia*P3d;_X&_%?H5o4!{v!_iHr%xI!vJxVM))*^tGLwW+rxr*t;_Q2# z5!H5in0$;?Y=`Cc>k>7~UJ|#Q6SgIi zN+Y6^bH#+y&l1zWe~vh9LaE3~j1K#&YJ(Ut;{mav#dd`LuPl0Yd@6355Y7dwG)9qL za<;hVk1vVk+v1D0BEsEQ zD_;NA4@6$V?IkbMIK&?-ILSx?LniM2Y-q zKN7F5s1*+1-np`J@!~Je6KN@h;<`V)FPdz=BVK1Azf+Wb@?UXYK}1-F4RPYEJD(CY zef#GAgRV~T-XoWYx(G4hrhkhni~F#j zNBz6s{qA>1b$X;_3orcpZ#@0-+st3Qo=&%fPOnF=*P{_WIvdN`vT+^jw^Sl0k6~o0 z=@46K2iW@Ad%XPeE4=l=$82jFK+@{bYb4y37Ah+%sBRe~u^^j-2t6t!qLMO6jttV( z+=A5$T8$bJ3WnS3I0tdq`?2*|>F#W!p{AUzn>JBa-$%;mDHJ7}zNjZjY9q~8JA?)EUyvzz+*W*nMyF1+PF9{$N?M0Rdq>CPU~#?9dRTW;mL%g$$VX(pleHT>`I z|K#O&7f{yWK&928*Q*%pZlSt-7iE=AcnpbTXY9KRNu@E7UNo9f`AMi;RxGwb0zyKP zgLvF-e6oT3(`It_gTLm1TP~&`DdMOq{@(U2y!*ymy!+ley!p<2cJ1y*XmsfHYCN`1 zYRh&}wrdAfEjFUki%5zb=G%gPIx07?rO}4k7|YlTZ{(KCrjmFJk2#{&lbln`_z9y) zi`L-m>!Gc+fwHX|S-)-#E0=xF;)M(O?6c2Uuy`4pt9wYAcq#Wi^dNWq@Eo$fatD#0 z?)r^rtm`Ud*oqmyW)E!53{`} z=xOEicjmIK!+|j5aoLU2k?Pklciv|#S-g;s=gncxyoFTQV!8b8M|kL-YZ#vybHEXB zwIQ0^l5vz~oAC_vVjXZJBO!eS{xhJ&lZ^on8LEa^8FPzdZfI zTP)dJk4tFK>$S*U8;w?$j#$tsamm`RlO$md56dZ<|PHlJzHAUU=)E=ajLND?wr^O-6Y#f`N`Es;erf z?{E;CJ_2Ls8eV+yMP7aF9hPiqz>NmIUPI7jrLn4#%IaEN5jhm3Mx$0qs5K@s3vw~} zEHty zR(7mgK`$okE#++5w3S^Il~k2&XW9Jsczf=0dC$m74{;)&Uq_S;<$6(3KaqnfIg zUc8=unkvg_8ZdC?wfAu283sOo^;w>N@okoEZNxn!zadW_jg?{fZS7Z)JtC9HJzF#- z4N(bcq{d2gHn-5{^x^k7Xm4r7smtKJ8}H%%pWMjw(kPmDZl|H!fzRW>>e6xg4{qhE z>7~T#)#%Jgm}M)K)lCez-B@?mQc>GUbm>Lheb3EI%85aZgw`0xh_MqWOpU;0?ZGza zLm(jseYo8oFeEa0=FR--v0rn`MH5Ll;H2AOSku8SHg4QTUr5bB(=Ik|-pcm!Dyqx3 zv0}j--uhrEuB`KT@X-gj>U;SZ4|&O~gih1Pnom|>jhw_?58cM-c>O_kMLMWiJBR;0 z`yy}7T}FMM7p+c@4i8<;4b<1SFc69+J4eg*IkR}-#aH=o!Fsw}pwsJ6$!^-~tEsK2 z!(q%K&E#X_+-Larv#+pVeJ%EY3cX%W$k|IvV?E7n-KgX9$x1MyjZCIw)JQTDO!x*m zsoK7Y_3PKMa@k@&oj;E`bLO&mT@_OHS={^ZL)>`5L=q#08(#@SY#PO5Cs3N3glubJ z%Z4?qTfd(5>(;Pr;m5rH!92Eg89DFP`+4}D>zS0Fu$L_k(!OH}uRi}QufF*{%XYL9 zRO`^|Gz9IPR996|QPGN&l!mHp4KF|U60_cTk5%Pu_#_Q_y^i2u4-M7j)HJpuN9T~A z8i^#S&_^XxJbpX{Nd_F&9{Qa=grq`35JE%Zh|{_0rw{YU&#q%^#=bj|kHjCdlrLxQ zk_}i~5~-ul_`3o1baKWd1G5X_?q>Ujb*$U4 zkqztCux!zM-hcliHnqr{dBgoYa^Fo%ElA?PPH`$C%(3WvJycb<;Bb4ev{X~xU?t-gzi*D^ZANU*1p#T%PZ>tx!>HnVZdHp(ihso1fe1#{lzorRTTo_jkFJ@6Bz7bkwL z3&K@uJqbCZm^`tR^k_BRb=z36YAqW#Y+%i*Wz3)RJ|8XKgfHz(e){ku+sEV1K(p zmKH2dA3?XB&fUAQYtuRJ`n&nXJ=b&EsAM{~ZJ?&dNzgMuuT#hPi*DeuF?yE2_Y65uYJ~a8X>4 zi#h7>?q1PEC6SYE#?jSEyJe7I&_i!$8(j`9W6ryQ`|iJ+nNv%VI<~T@&O#vQq0j0f zdBR0pckTJ4#Tby(TB76QaJN-b+h)V->Z88AoK|NvGj6_zTQ8bOtU&{k7IR`Of$n;$ zTP%27c3NsGXs}3(yYx2hzW!V?qBKZq9r3wi7+0KyK47QEI)E<(l92IvTzCU2QcBO^ zwrqVlB=-5Wb9ozOj-+#w_x#M3C=jrTGd(>LB zYOSid=X@jD+3^LT7A?+`2;g3AHSWe;6>rwo?<&t9i{{Ses?2?IY%40Qjg<1mB}0A^ zy5=8_m0RAtWJvOdU1-l*m$uhtaOTF`P^{v(;i=II)JrShFQ+g3z34mpt)#0ht;rl5 z8*tB7E5w)GkhuNt0F=6ye@HC4wJV+6A1|>&=*z3AX{ctLYA|I@R%!E@Bob@*6vQ|1f0wZRASsfckjkq-$+cQQXKRtnvql zhTKkQI6T*uqiNuky|VQl)U2LooL3EI>4@LV(ai1%CCwyf^4kKlPT&KhuImn$j~80` zte?T`oG^ZIX=Oo>B=twK#P;En*Xt^_*IThU`|7)uo6lmu;>@KF9vKJWLreCN++fB$0g z*vPn&!%Su$Gb|fVA%P6R^ACZ=Oh?O)2>J707o-pO)$MOBxR9PTCE7Aald*L>Bc2~& zwoJW2Y{Qr-D1{QG^C-z#BY&OAr59gLnzfu?%mLCU*WOOvq2gunhcKS+A%;R;*hdKp zU-Ow?MX7xb8G-Lay6auXrt{oSr*yL#0ppQ(f*@%ZD8JVmsmV_(7Q|qL+SgTb{_8zk zoZL@yspzDI{zFB>z22FgK|a?E72CVQg|j&uQ7zrm%G#=u`V~bPi|0jfpcOef4S_(g z^H6q0-sbsQS5C7n*YAK$JPrDnG-RQysZ$+d)!O#V$bbNtqd0{!wC^%15zT~T`{fP?on6pGz%Q>Uj?g^LAZ8e@dWHy&W zLY49=5x0+_>NP`uh)L$R@0=QnrjYKWuJw zUP*L3y!~GGgseC7|F8fooI)XM=8fJ52bo@XAKrG>O&|e3gW~>1s>}s5DpIa=qjhM5 zz*imv-#Z3DeZyE)q%(B)J_S{K%!5vk`tA-wF%#P49vwliPbhn4 z4ZG6BQy!C$lZsIn2Gd-ve=lf*=cPTkrPJV)Z=v}NjE4j^z4vN1(?*_+u};|my{xq2rx zZ8V*$R6i~r@D`HtCAaSmeTEPu@E36Na5r@ou{l}YaGPOjcP@&~>Dcr#BX3VFbDg`7 z2Ar1D1=;5st30+A#V0CV`S+^~E$ z=&}v`b4!x5#$c)=9$&lFgCjqXHxv2LPcob#CtJun%kc9#GH&XbRg+2-jJI{_DEq5> zw7M>EG-AymgqFdl+XncRsk$&5fsq-r(UhY3Gw`SOmEUMggMwK+QMZEZ=$44OJ0@oH zrg)iUG9Gp?9-NpSG)3`93SqfL!HQ__L}G~wDO_+|M%pf0`(9?^Ova~6kAGgfKT4yF-hB%>Prky|`$$~mc_LqxkxWl4zxgZ2g=2DU4o zESNwy()OF-0mImaczNR9sfoI-ab04NBH?C&)1@Ke^iK;=e_v_sp8O+sRQQGLXTWW5 zSEJW=0Y=isS0e9IjV<37>(8fKzF&2Xq@Ycf9gn}eW^tLzRC6dL zHO~VWD2FEH&89zkR?yjLdEEA{NsSZ#vu)81jSQ)oI|SiT`=k+h_bw8N+kv5y{aAhu zPzo@@b2$&KHE3bD{^(4k(9jAs6)ln<1+vb znqb(?IpU}^e zZXV$XmveeoQm7ZfhgQeB^{<+Ou8wGjvu_F@DNL4_zip5zjo)$k%EBhoAa|#fkCD>N z(G)xwpSfBMzjQsgIn3mHQZtYEB@;@M?fGb}TD@6@OO>_sM2!QmXy7dFVWeez?5e%J z4&}SrD&q^R`Fs7}v>osWCGKi3)Hoke=A}|IR0k{OQO(K#X!i5|rMs{`FeewM1(mfa z0A)9-|j|%Y38j&ny8T%M_D;-hH$MrXt zV%W|zvbs)Cr!f8CogIaq5wqnD4+(D(Cp=BMkr#YQY;q2lgRHLWJC?Qnz_)IqE42hq{uo8RNnhJA6J`{QAfuY8<}c zIr2N{#O+=j2G};j|H7?;zPO|Pd!!;tdYXi@G!4|MHDGvzKuwWAMWYLXz10JC9%l}g zI26MFUW@+zaSir(vc&R0MJY*2r~lfNKW^W2{h5|)`wdxi`%9ql(~V;+BGk$Xwcozg zp0h0pmN&Umm!41w}VMc(`O6g6=#m!Yv4&MHyN|pW^7h zVP+j6Yqq(Gt1k6%ZLD5q-Ge2}W z%xiP1`y1&Ml*v+)HGv@g`AQl%aeQ_dq64Dy^42=yR?n;nBnnJkf5>`iIPvjNw}-Wx zSBQaHk-V)i9@6Z#?!-{?JyPkjZ~i!@sxDsILe=5+CS|vx}_i=iJvN+c<#w!bXHovF1j6ru5O0LFN*1?jm_mpt_BPyurNy+Q1vO0T0&()r6tCuh~PuYOoqSsw^m@C=%iv0+GNxd9AHp*K9uZXW$TQs`pZU&9gek>P(OZ#$PjYO6o5 zE%#etv{`?h8?s=v+0c^QFG0mN7pD;A=B3r@b`f~+51E`FlOr^gk?hZOrQZM6(xT&G zw=OB48ydB*H!O#vXsp4S9MNsp0LwhPG+F!K8fhWPZp7g;rYFB^jOMuX)Dk*2Etmmm zesdhCBf72s07h`T#K=$+C4=6BOnKP zGAfROk0F4G<<7*q(&TM9kKL^^N`}ttJ#22@vJ9mV(WLd>#I+;xVM*PFAp({S1j2y< zgh|YTnCoOUbku=A zyo}cUn=_)0wa|8_V9Uc9tM`Q8KqoS%_u`;_lfYCM(*JKq4)u0>Y|f#BsPai15noP! z7k$bkrCJ#1(A(&vs#(p`2Z+a#P?b2>YJ(_o<#Amib|G5 z-0)a!b8hlN%$U3^$udE_bK8`%3(8SYdTk3O?L@WJNeil4E^v&dsL2ab`(!MOJbr+` zu}`BlV|&8`YNqaLBu#BRw9apQlN^S6y*_kpvpekGg7r>Hg|#6lZgCL8zJ_3MZY6X{ z@CPyKPes+V?7?Sl1f7#0?)Ki++{b0;DqFl0-0OnqC<+d`{32BEl2QN~zO$Kqez!i~ z%hiDAb(K4Go!}FW<$g|NtE%X*fYG1%t3hJ!-&46ID#dK#DETQNTSCH)j>L?H$KL{F zYS@et6v-H6ObV)Dau)`IPc=fvegJL!*zuY%6qGpZO_mpC>hsgtP z2lpQT-FP|q^0(!A_;jvWU;QR2&7Tn33JA89nKUH#rUz}M^={{7g=HzH!yW`+_FAq*$(QgGq^4(+7Xg6}krW{?b=29>=o)N(T!Ok>(fd2lkLJ z*JCW01PT^%L6=SgC6C4@4ZF85$cqAE4aEy5(Fw9Rlq+ZnxE<@Bs)oAK_Nr8&V3>y` zzntE4(j`^&77YZ;NcLT4myG{#$O+mc&e+BzP|=Xu{MBf2>616~qd1IrM-#xrXkDpp)kYjQb9q>m^|2a z%(PY$?ibp$l3s6b31$Bgk`|&e98W{R#qr3z7wzvaUtSY=X#gWaTF4@vnv*Oun9P2S z>|O8SImd<5qbZM%Dr@7?Ccm(AP!D}_dTKn2?*QRj^7VDr86E&?KAKwpV-p&>bifz9 z9|V-J7PWLuv>cYe#QT zDr|>gDqtN>&Pxso$SMlI<^#Ri@D5zjJf5`7RG%HM*ZVQ^|61^gX()cLk%zACO=-77Ga_GeB(4LkzwP!g4;P$aED7iR zCLk8RF`zWbZBlMC6j)uVpe$Dn;*uKIdD%ARAWo{VqyZ`@!vUq^sQP~!_DI3LE70Y2 zKZc$Kf_!1Z`Wlocm`gbG$3$EV18xjX{36{_Wb|3@!>zShMO9tIzZ$A;T-?WirC2H+ zz{U}L?C+-%OyFjE(e&^7B{G@|u&5*OU+%sv&W8j`4Bfg!!mApubU5w1OpcbDENW{C z3pqzf*)^TLG8D!cPhVPPAG{b#e|R)HIVs&{Ku`T45g;PSlU+GTU|frU+GpHs;>vvk zg_QA@NXkQjHMXGWsVLL~aoMX<$brK`Rf1A}XIg)6;^`e3?dc{yLXm=k?>_30CZ!K} zX@TqT(R!HFH#fBEEwC3ooIg4%b_t7%6N_MdYPfpCgP6#>bZg3 zLl}4@CC7k&Qp{AdVYlJ%y^l*WHv=rvp5!X`yHNXaMUnWOWa!+*`>!Yq(i92@q0lzr zA+IN%EbzfDstK16NQDc^S$~!e|v#0tHgr)LokdaW7OybGRLyC zpob$IZ*Rf~066g5GoWGAB3ZcJMs1k8eyZCb+B^iflA4*RiOn#fxIE9RYZL5WDXN4e zB}eY-s#MNdEdqVVa^p@5Oli$<)x`El%tRaF2vSfZ+ZB5BdR@HitrL>FU|(W6I(uuk z?%s@;qv!#PIibG_2VQdh{rX8nyl3=TXxZ^Ns!R*MNoci24r`&Lh)MAe8>0Mf__vLG zGm3A&@i}Sx?x%y5lPVk)P9M4LJKDHvsF3*FC|c;0XkiUR&i`iXJ;RG4^%7vH4d8jA z;{V!Jog7W>&yRl zwGz5e-yir9%5J)|_h>S6|04P#YsZ8oXc6T5$r{MK$pixd8F_wpeS3SrPcR+;%L8P_ zSH6hfPhBekauAS zco`CU-ZpduNm#mU33TY!XatgXm=FteD*B-;Bc|y1W^KQfGV<{I`Whwg?{H<@48BB@ zM8ZbwGcC_?S=&B6nkSyVh2@tuJKi>Sei2K1e&+UWx|7mSE=5z6HRNxtrSm@=1vaSr zVkPq5xK2+dz#fr3?(>Af8luTR9|bi=PUUwGe!XqOPe1pzzkkl(7st$ncV+5b)dWgB z8_+#Sm#LcZ0Id{LE^uD2Bhy!QbQ>moS3^{3YCkf-)15Y6!inp6;|QjbY7xg^rA6Hb zz2oZUx8DvWdk~2!?U=+M4`x~XG9Y3Bie2qVFy%vJkqKK{VJoHlafAYxrh}$Q6VKy% z+JCos(%oVXO9Ec|zVQk91`7%Dd>?r>yq%#dL-8FYbU+o4%bs3h)2ug*{`^;i1b_o+ z$Df17D}9wzkn?+av0dad zvyQEXO5xlo(hF|ndv5oaUX4igC*`5k+6r_f2PiUq31p=v7_peP-oO-Jm7=d9F6)JF)Wp8T6H)Bk!#6Y%+dsJHC3O|R7dwggI*%qT!+ zYg#uQC(QAKirdJK*AZ`j0?#}WLK%&o2?Ck2l7XZN#}OvMwx5VIPxT8FQk)qhbO3IR z4x!_QhL5|%iiO0TvV{?pz#>dwQhn>yetPeFhNgz}`};vPw*VF8M#^2afgo9mwWdLT z)lAi#tvZ4I;c0;jL*?v`uxvoOOz0_h&L}!rnVQ|7`b54&;A7? zV#CkUW@k!DO57ptS<~Sv}xhyV^D#x&(#(C`Bpt$={VFdYvsu#!R;-RhCw6Z$Nj|y)6b%xmOGQYktwi zO_CoRK1MG7N17CBu-0->LT7`KDqDp4r3jKH7ZKTk<&IWNf!#<5SmX7%e(h$jc}e2J zO4uJhDF#MGi`S`NEwnOVT<_qy&2BIC%_nYIhE|8^1~O*@hg_d8J0?Z_=GL`FSq*#d zHZt==QN--=6>9#8j+#{PyU_OJhxCtia_V2`dhL~Rt^Vc{_7Q3RaofhteH@yRGF`y<&wAFbzC_oNnt5NRadNu11UA-Q#i{ns%}#vlSUv5@ zU-$ha6NzPMJG1Y0UDYZiZ;z3g$oOst*HP)f4PMvrj$Oj9-zKsIY#Aw7R+uT-k}F_8 z44x3fbVr^A6wVsz44%(b0MAh?FCtGmh_f-UTqGR4`Ln zvKa4bVJ$lp)p_6szM1pc&U8lMS}K%~a3H0M!~#S?Vi0x$3~BaTqmOGuDO7s>c~2XI z*4fH2kbauUV$XU=8^OO@5PdGV^;ic%fc#5mBhN^tTMRI4kH}1=sMPN4O@oA1n9c!0 zPC2qD|2`2CFZk|)kbdb-ZZS0`ukgU$YtAXoz_$Ee$x!&H$=qp2$L^F}_SA;hYd#T_ z@`Y$26mUp`#P69S<;T`gP*CBf@_hpZ3+O%kI*?90gkB%uapfJ&n9mT5(E5Uq;5_>S z4xV}cpvXiFhu{tTF0}>e&crGfjfJGqfMs^IKF3Imc|Tl4w^qyMb4PL5(r>cqv{!7* zm+%7*FJs*G-2S*xrXQjbn~PT`!@hL45pa5}oSW?(=DCHZv|lg#wjK04i`!x#D}iFl zOnmsrb=U0$RhattkFh6dlw^In?VyQ$K z!kMxVp>N*Q2M_Z8)I5EJ5dY)>PnFUBQQjh-dc*BwU9;YAN!L1elmaGVxTk;>a0{2c zwlD^a8zLYKS5RJr2bmPA=yUy3i)y1Ys_nS0(?pPyN~<|fJ|oUdY3dy?E}nvR+-dyb z6B%?uXI*FckfOM{dI-^Xb!z%Sbyd&)1uxT>5*^QBI-lp2*=j>p6vb&piUB}DK)ZgM zl{{j;JACv0_jo5gpI3kjCJ#wI{fXM^y)mWA)H{=-KehPlBkHvIPRZImOAZysoz~BR zYAY$>J{diWbB~Ou;}fqeeu`3*CW)Gw`iEaeM!|^4$|YU~gSNOiw|3;n)>Mw7ppnEO z1%^2Q&6d-2u`(?rx}Pfqpzs=iAOg*sTUkqQIuY?;S8Z1^VKQ`?!6Gi1nv^4blLaS- z!xfQ|EU{lQVyc7&wXNTLwl&rNv}Jh`p})w^zHo#T5m-2nlAa_Ipuj6lQ{49cNX#n8 zarjR}VO0L^aj5HK8jThY-|EaAC2gR0UUD!sJy%vog%^FMGcJ2U3e|r;S`XHCfm6`u zO`lVZbGEG$+mZSgt=bxFO5@$Av^>Ca{zXsVj$pnxjia>S+Wy~#t=ZgL!HyT}!%)wpOtZW z-h^eIV!DDrCA74*r`BigsPMf!I;sRbP~Ih!lq@GH#LlZ}^l?;wvE+8nEJtqJyIwB+ zTDTsI5@sydCac?CZ*eio#eZH)w-i_gEx+1~w>X+?4imDsxEnXG!c`%FYVPE0!^!7p zbR4#-uw*D6fcQz;U||R-mDo6Lq%rs&q`RWD4GA)ykATr9yTGYH_z zj8hai{l0;T5wV9soq|w9h1!3FXEu84Fbu~kfp#DBc+$LA#8h#3ckL* z;@m?c_fV@r@2a3h%~tfVYTdTK)~`K6!VeVon2Gb*fcPo$`bOv#3Z6$3^Y<^#{kTQY)$sX5z>AnRH9U?f<`3D4d zH<+3Z`CPpP8R(91`2YXy_e3iN78U%jcAY0qlLA^{ z|C{+yDgW=|Z7c691qmx@K#T71#Y*CWgVOx}Z{Y({qoiGoR9gjc0O^VcRB^l};QY=< zASmBo+@mBxrId(T#`IUHk`g9bVuRUr+OF1{V!JcsY1w~xnGv$oI|E9Om~Gwxl1Jr6 zi_~vtm_7Ie?#bd|;9ehKVb7<%scJg+&=u$_xrsJH))*g&IQI$?y zrbRCS`ND}>fDs1n8RD7W!(%05?w`Er@CkV|f4T`bDJ*msw~wc4ef5vmWk$V*;6;#l zyF7ush3KL|Pa(;%>%#Ver1>i2sjlyJ6`3>>wfTJ5oY`P*f`}}<2sKI{F}GV>;rubn zanB|WImZ3Y=~ZE^3%0s@C+>A+^aQn0e+e8$$w(nzuxpeHwQ%3ni2YTeg(l{H-k*|p z?k1V7yEM|{7L{&u#9aLDxc4mt%0wVss|^WLDWQffIgdFjN_lDu z9vROllkh(CnwVUhGMl_PA{J?37>JZZ;B8y>w?vNa3)M z;LrR>yr;BQ4?}F$yIN>u36A_VJZsp_trz>DJo!q?CdPf2WWN=m$hz8!OeaeY^Zzur z8ps)xT+YI8S(;zKI5wDBes2MTx%`MlrX^HnVHqsJa{4@2VeMw$VXpec`j`MO7;q&p z&X>o1j>Hf45_=Q}q~C?{{EGV?Gbv?IkLX>gdyA$xvl_GcbkJDUELmh?0wne%_5A!t zk7jxwbDqEWY_0u)q{#mXILrCKkn7Cu6@1|>Juzh`IU%^KjLrxf zQn{P!YIpnL(t!%cj-i3L5d#yofN`$VeHv=N>Wn`iyr4g#?n;f($!)q_jTO1i`_g-j z73+!e2v;Gxo9FFVAkj`V7uW8xh0Dd&=LoSwR?KPvB+h?{Xdto?>^wsOugdec!xFqk z?JCe*71{YnV0tk+T>K~NPHx}&_T+ft)^6xQv(c=vEv>1gJidWE29zq}Q6W{pP^qb^ zk}P;@X+mk^_P>FpDJ$}(d(O1_#+J|m&YYsgx`R0?A|pxV{yFT&{aoHOS0ey*#*}_eq>ojR$8v=Ivthro=zbrsBy1lKzgnYKg$frPD zp7>5;{}5vTUX+FC!>fp{dyi)2PFr-kOa{|~4hD8zcfFStF5KK=3~7ZtuT8Omh$rv+ z9-+ocO^Ge-)|-4w;)8pPR8~&IZsFgpnRjEljapjEv2)+-)z@hpFCk39_n5lksfa${ zR2zzWi>T>gyL+L6>+hxW^8K zAl|@-YRv|V0W4a4A7JM{u%C>0@}h$`&2LiLfaF=KL)I5x%R-J{mY^z^Mh}~=41sd)Jk?m!#&edr#~v_Xv%_Y)GI{;h$UhTbV2}D7>&&NZ z>#m2X)V(74`QllU$JpxjBEFZgImn?ZQ<_s9jS=g%S=Uq-we-aIfMqK!Dd6!D{75zZ zm!`3vl5@P6{KGpTGD@B<`|eV7dFkPIIgu5RXvlm;Zy?P&m4=2ec(nkh9$mrhgRui| zqQdTc7*9>L)fFGHlFEzJfC5nV}oUBf9JMPMU2M$!xMkL3VpWE4kHtP!FcB7P8dCqeLi3Bb#>N8yWbh?Hp3mcN=|Ew z6UO@|vk#i5Uw1cl<8am*J$8QA(p5uQ}rv4odv#Qet9_xV4ex?wy#jR>{#>cOXO>*bL`kzPD$0V`OeXD z+O3J`_QH%M1)q@+GGi#rYcc&RYi*BfcxC(rhUv1(Y2yp=<7ZLal;+Enfh}U==POQ~ z1DErP;pe2h>b#ZV7iF*aJ(FKYW6qFckLXxaBTlv;<{gboh0>#zme3K;=Y8gpmiCG) z5Pd3&hK)P$P>wMx0J`+WTD-zfOXuuwg9S}27K8Cp89ZLA->?uLZ~dddG(@YuxWO;5 zbT@WZKHyT(k@tP#6`KE;>HCQ#b~2jPzxZCkX+uLp$)&ZIt`z>5#u02b8r4<10NOgS z{Nh{=;_mS-_w@6%rsPvj{H+4Kfr6)?snwCAjBXs@16XilWZ2zwL9$;9YA8cf}a*235Eg%c#-`k{%(dfxwwi-Mwat>{$N>g%2jGo zk{JEh+XET74jL5B>$X`+o0W#sjbF=~W zd82>6N&EdX#>99QD5^3NZTrRKsdwRC`-T^HDs(;NW|J9^!%5em zhjZ~$bP)8Y?qfm}nZNW7wYOU&v+Wt&cyB6o;k(7Y9xz3pDNLZl;hGNSeQ7g7gEWzO zKp%YF|1e{5dHpbnXRw@zdps1D>5KS z0NvVtR|%F9XUb3?BN3l5tGkB_F5)g@?Yl4TAEHReQNPs~QIHLIH2{&B;NzG#Lc+nGneDKY3c1GAyj;1E(EL+p=mlixZ`YFQPC%r}hCLV`CeWPLQH% z-o-{krCHrC1GuQ=l2Kqvx)OzJD_4wPjEV-G7k9gW#7Z5G;`n-{?u`R_uA6RO%IU0I zs-t3*_MF0nRnU`pL;5j7RdD#TZL%5?A1`#RSj?FVsUsm2-K~N7Ubu?<93jLHP|}kO zWz-LR8cP}yD%@m|z+xlQd%I9LLCH z06Qr#`d2|`roL@UaV~4#Tz$+B|GZ6H5J+sCZe3d7gWe@>UiyR4 zL6*YS!z=6c11>A^>_aN#pV4<&O;6^-U(qF!_`TdB)RoOyla*;02k6+P#nvR(HStN) zX`?Ss@6n5mSA&*vg>5B4ur)3npaaPr-M=-b#3jZn&_3jfdvUwR=+BGIVUpEUw1VM9 z&g9XW41*pq_6^7{9oj5!Wu#I{l8+JL7J11l!%Iv2 z)~OcF`E;=Y`a5!nnqE?+fz09N+NBhQPQ1PEBXoN6C#3C8>@<>yQ79h|wyU$5Q#|O6 z%!IA7q-M|{md&{h#ZT+{x=xnA)E?kW{Veexbx3Sx3_OrL^RNG~RtvSf~U)bT{ z+8e$Gq=rqc*A5&?x8 zwn|cEtYz$pz~Ts^mDZyu!U6hh_;{VffMc9r0BI|P;o@F;nPRcJr7-OHZ#4>$3a(9w z423F{EXNPxgIUT@e%yKv?=6%WOxly~*osCr5jV^~UQqloZMC~9iB!kjBv-K;o7SLG z!ektUPk7x}aS^lsY_5ClWtThWc?y-1N6@7Nl_VA#O zC1u(0y^?eJdW?_2#~!|}D%Lljy!8b!A5uV=Ff1G?9AQbd8^#HVyaZ}J_IWW9tqGVK z3Oxz%sv8ED0!4@Sr28p%{q6O$4K><`(DKkV;2c!)2y>hFzHRzmG(YlCT@rDS?|Lgr|09M5&yK#JXxB0#n zb&#~#4-1b51>WCD{*SVaGA^FyZ_A;q-c4>|K)uMbqCaZ?FWR9>2DW*aiqcRdCG- z34caLlF;K(G%y(o7cRqRt0A=m-DK&=Llw2?{Vt(zzob=1%}_ov-%bJ{E}v=IVMDH0 z6cORTKN~{f!Mn(OjfLf?+GL>m%A_7&(7D|OkUO?X_2#q^W6KptikET4Mv$eVVw=*j zFK>qfi;D^whw9_2uQz+Dtys7GrOq5L|7Zku;h%V3AW{5dBGEc!O*9$0bqCU0JzsSt z-d;Iy3N-9-{cB`uC9gL4YW?Iyn4&Eq{=}MCA(l{1y%F+tsYW%|d^BDiCwA&?!fokH zc3-(%)95Xxp&@fs_XT`Di{Qexc9)~GGZ8zWln#(;E@p?+TxJYx+e7QY0 z@r4!Cbr9D71rw8A+Ji^I@i4ot!+4>|I960Wv|2-V^Dy05G5>z_leSbMJZ<3Yx7~wa zTE$fFd-qW%!#Zd%hRSJJb5qpu>WgiG&-`*WJIS;XN~4wF?-guXh1bVHI}1l9T1TAE z`pA&*WZY0g#^VF3v*d$_ArkSEcQDS^X7KERA-rsOi`3Ih!O>qZet|2W7#PovYhN!+ z-tde&ID8JUqWl+$(>b;azdgTr(ztN1&yu#F&7~=K6@P|u_yLHH@xjBNFOj}sEx011>_PhhV zZVOW8_+sE$a8{?s%BO30I8?-;5Bu4dqa99cEtvHYGHIn2L@r6V;vfpYQy@FRR}uaVt@>zx1cnpFRTN6p0wdx%#WPl1a+J0BoGDN@=|$cytckSlK!11TBO! z;Z~QZm}>G>Qcvg}5i9VXA!WQd=wC=ggP&PRd%rxH@a}%}qey^LTCA>uaqyT`SW6|i zc6++h_lty455`^+ueFal)g@9PHpEqlbSpJn5@$Clr4i~uw+mQeE-B!UCoC%(7Kqcn znCO9GwN1G~Y~&Bb>>p{!?lRPHTump6Zj>o39?>G7LPtGIug=l0EeKHny#=}3RP$(W za>B2Ec=pP8k3jbrQMna`YiD<>3#MwMC3Eu%Th1mMNpTaNjjj*We`nCu4e%7{53{+t zt#lzk1Q;%2$QNHRQp|5!1y$9O67J-3oZu)Vu6H890%U@cqUAH2#lp&3gzPbzQJUId z5v4vZZUX1cPSU?CgtvUKWt`Qxl*OpyN}4}n220e`ayL-9j9@z= zy+0D7kwD)0W3^bs%0vjILyT~|iK>!|HB>1Llwk-O>xO_v!i_>QfQS8Nr}2G~l6=vf zVsyMF*i$ah?J?!UJim({(jO=ZJFZ_5O?i4vcOgjp&y+8A4%_>tw4Yt zoxUqagNmv#JVB5z89!Jx7@|2(n+p%FM6(X}tGoDBPxowPja0R!?i#(1+7@@HdhGT{ ztt`~6uK6t23JDV%rivw4h&b3)kkNCia?^VcXx5P)%vXUlnMH=Zz&2@yigDS2p%M)C z=#N>`S(HJ@&Q>`M6wYcW$vm`DZ*OW^1~GMe2saIyt`I(Q##*|N?oE=n@IPR=O9=9+ zh%t9AN=hnlrOH8_j?5k6?h*N+xE9&S{CN~8kbEafDSjAqTVJrgb!oziPzd8PW~JqY z(u@%#8hu80vvp?zD?(*d$b~AgNfqD1vno?ym2=kzRl7(Ef}pK%w1SWueyR zC>f@OT?uK&IU)hX?4e3JeD=-D#B}x`MUwi-sWHGVV+{S|@-)bE&ghcU{V-3U(q{)y zMRw|s7aHl_^$vOogqr&8W(#}xiLGg@sejpoNbJV7(gd*c;Ev~0%b8+`~lA>BWK|rC`2)Gl~j{LzGp5dTRw=-h=`%!pWg62 zsZ0bYuM{%#VGm)ZS5!JisPCHf9y;XhoPpSgHHU|+8v9FDih9V9D=V|9E7ir$|0M(^ zF^hY|i|ZyDR%CMRjYD>8gvV^Sl~^1WYnTRyb|Q=Rlg)ClVh|;`L`i^uqjMYgQPWr! zvHeVbL<21_2`gR4Q3|pk3-%?o!m>hK%Ge7FbgGmN(h5_hv#XtVtvW=Tw1bl3L5!f_ z?g0|kN~J76IBDFW0!?mc#LL7-qTBUbSz`spmhU^SIyQxim56K5uLDaWw^aXrK~bi{NxBQTu&T@z9ETWpi;))Mt{w93 zm+1{R|6;|VaDa_Hi4~kHbaJ&?o#Fg`k4G_nw~+iJC)B1H>e8OCSh05twzCv^gNDc+ z3%#kz2g9t4jG-Dfo)`Z|cL>T-H(!zH#||KMgR>h5a#$)VuR4Q^ib}!5RqUs}A89Bq zuQ?`boxY?^0Hz$PqT9(9%qF?m&ho7czRuLFbsF`Qf{2SG*eQpGSD2ebZ@?z%`N4g4 zUBhqfDdy9q3dV6!9Sg83{2XJf{cBiQ2{$_~w`*I7A;Q`fq5w`UImu)7=FzdypwbJ0 zW5QENF;}P*CIeB)N_at3e`e-2R_wys`cY}3gdGoY!HKCs5%YEc3IoaZKe+l=xDi`Suza$jGq08hQ#!j%do z?WE|mCVDmY6H?b35iKGp8AARG%Le4K?0D zUv%f>SJ1vCRnzgAvYr}L&1y_l7yPlj#y8I6^rg`$xzNAe2fE})_9*{oMg7Il5wHXr z;OPoj)=Xtr)S}Pt*((5#o&X@>CdzCr;l@E=V?wCVa;9HNSxMn<6}a3>}hO|erUlnI+!V&tx}F3 zLVnj54N{p}pNVT&OFcNxNhLE6Jj&RQf7pMKwrU#;2k|Joq0?V*+cJD0FdukF1=dJg zJRQZ|uOo}apO-cqW;u&HsK2kMHY&{(gb%Lb6Ox&t;^V?IVGs2-!PsG;yPZ;|9a)l* zm*X_cFi^Wk7&>vAbV!iQP9VBe9+&%xgVsJ?n)>3vtWAz&5Rfc9*O9O3!Of@O{|o8A(nvwXz$t0-Cwk!i z_eI15;7qNkk{P|IKjEyNe%Ki{BT2c4uvQ)EG3Ft>xjBWJ#I0BxHj<_iJXLsVzfr!4J6$7)`+jlu2H6JkFy+MYvae*2ku#tVrMu7K zz>RQz9%B%Oj?R8-+Z^;4TB6oB<&dE}GiK)`FaHhCyOvm5$sj4|gQsUk#z*OPFYZ(! zL@@OAo$UbRPaJGT!_ay7sQrW}9Bu-Zt-$y7ju2a%2&A~7Q*s(Bhgs6MHS-PXN-^yG zzHLB;y(D54-MCU6r?7T%xJ`QV!Bsl-^t5cG&&HFoQ$o&n@s10F96J@F z&QL>>blWMai8w%&p8ay`ZVzNV!DXw9p*kW1M=!^gYTHXeUa#hNg$7}L!}yr0;S5q? zFxC~UOW8-)OIc)fRZN$Kg=9?~#L{W-_E_vSDK{}UFJ^vyLWcnop&jS9*QIm|(sF}@ ztTr64@urH=);hyR_HX_Xh$3qL9J?M7HLUP#mDJZEL+4!XKB1CvOzI&MYJXlMKV$dm z2WSFb#`b^eU-vze{>|133$Z|VTUx2N+=k-E!H()PgthWFg_|bi4T@xnc&jT*JF#W&85Gns!(cK(r*4K)ZG^L51V%pmI3Axq5@BAB_JFjS z>|{>LC7g*rfm7!$BfCtFX`g4Roig2xY54R*n|sP zR-od93&er|*T^yW$1C#@7bu3Zu@2?=X*hlK5RRQnM5!nY&oBNGUG5~3&1ehRNYHyu$vj(^kIZH1<;5El+@ z#rj=mQEPERK#(sSEGm=~7NE3R1Gmr~m@w-ROdQ@D?b@{X%c4a3$xZnD+ugWPqlJ}( zAah4V-+M7}`fN<0&-t(z2}3DsE% zSoiG)9J^EioumzVPkA2ycxnPVddY7R_{xeboZPhqyG~v~d6N!iqXFBg#n=DuwjVECw|i>Z(zcc?HJ~?#Jh*Kz9bK^#lChDLcLURvGb7eo>p8cUL~Z^PHv z5}yT`PcVF4fSST=5Xdq6H0LE##R$hdHL+>w|dG*vfnYWEs^ zwdpttYn9OGOrTf>_rR{0yzq5AH+>*{o$O79u{jreRxQJ~`>&uuZw5gqE3V?(f4qP_ zG65o)3wlj`6YtFF11G5nG^=rS`$u?T$#xW~Md&i>IeflkKB9e{{=8^N8+1*jxO{9k zHtjxzQjG{6-kwm{R4C5QLY2uK-S3}-M`lmPfGz>Ii2+ul68YCpW6SzYIG@)HXP>t4 za}z^ZQH;Xka%iMJ=rv*r9(i~?dWHKz@go4I8BLX?$jiKnQzuU#{%jJ8>MgkM)y?SU zqrue!>+t2;y+|&og2rfpNbZc_m=SpPjn^=#Uwb%+1^Wj>OD(ulIu zBUt+K4LFMdX`2v?pZ^x-4faJ-Z8h@KF5t+4Lx@jIfwuFb*cR@Iyvy6Mvka9;EitG!r?Bca3y{xmKQccB#~mk!qs@=!7#|hfHM0eR(`e; z2d@@GZy^8!%C79f+b>*zOb{S<4aM}gKEkYCevpW4P*i6d&%KWzw{`22D{$&NH48M)aGBq zf$f`d=)w)utMxD%^iVe|p*Bm=uG>(|ng0YP4vmI~LUOY))~l7Mt0_Zf(it2&d=wY1 z=b_DrchNO6T^1Xrk#my(2)pEbZYb`%!AumGdt+ChE? z5N$Q-P??p8otrl!KBWSJvnRaVCHjjJ_uVt`ZBHt8);)eFqMnOh8tp8Wx*? zpnDf$^?UOX8Q=&|htjmO*mvwKu3k?;PDve%6yV}F42!od#h7+&AchtC<|foummwo5 z9tZXx!G#-@Xgl&Pbn7I>#SP2x$+!EFQ>_LZgE8sVrFiGD``{xd^i8F>cyt#w?>u4m zS9CaB*^Jy|yTAP6#$e$KPhwj6ns|m`A8#uaiGxnZJMYG@xA1^mhTZ5b%`OrIu zV%Wr4m@{P*y0&$Owl*I}H>|?;qnA$G12JyGB(!m`2eHkhLE*LI z*tBgw;;-hyr0_$pv9s~w{2A!h_K&m;R;a2=k(ZT%v+*YopKuY`RXRjXUy9z5uBfAcVDl^>9k@rG;3g@7_NR1n6U6IygYLd zJViPbTsea8*KWkgD>u)@b0b%ZU6dW!3M&w>OiH+;G;Bv7R zuD)&I<0_!BtO$iA6)-3Q(Ra)=%$+t4J;Ho`sy9|6nwuI>Rho-SClBGk(bLGPv0?1n z+tE=-$J$SqVBMiBs8L(s)P5jdc={m-#TRh!$axgjX<;KEk~ko^TYvN)I0Tc&k4D#a zZuY!r(V+198SLG+2S?6jLnrq{+41nKbMVoMw{efZ3^c2de>EQ4Hf_iGd>w+jbVab26t(&3xV&Z=v~FE6>4|4C@1g$i zb(Gu@kC;?dxOQR(zFxl%g&H~h+6KTy(4uX8Tadjj|4aIa8LFyWoZS5bHXpnKmDmk!gM1;@ z*Q2#_elr$Ou0;N|DVCkY1 zIJFF@IJ6&#x&7PV#PJiztJOeLSB?_34Y2A^e0c}nem1e?JDVbSbsB)D-uwsdZ7==% zv=|m6Y74Gl&(sf4X`S7UJD;zVPehq^}XRvI=TAaC24YLif(SXdutMN+u7Kj8u z;ol!GfA9i^hWS7O8>%jC#%r&BhBLX1XcIXa?|u9p#`kFZ$3B2e6Kb;0VC6^4arAl# zbQT*xJ@PN?#Nx_CIEVqcZzN_t`xYJ^5O~MltEnUz$2P9S+CAq{TGtG%!E7I->Cg+a zUw8xaALxNLcbFKpnN%oDK8@{Lb|N9Q1X_~`2AvMg$|e}#im1Wk@%a2la9>;qoTY!1 zu_BDkRk(h7FE(yDfb@C+fkAEICNZF@qyUBG^$>f8W5ncHc=(}V2oG#y_nXax#?oup zx@tWRolixb&J0q3#K{M-{fFSehv#8JzfdS7f}Iqv$-=>P-(&xoODL){!mHCgm^ybJ zW>0PrNl54$i*fPzZfw~TkNheX3?>6~Y89H)1~~eJW5kSkcw)wAbn>wa;a0OA%?$tNXFsqo3Sq;4GlUoOnMzuDkU@qARu}uW8YsS%%=~ zNVIivL{Wng+Tt9fmsVqRTp(P2W2(wrUw}O;7h~JyYM4zb6c=j&0J_R6So6={2VPo&NBVf$hXbyk$I(NFaVjwt;C5U+3`sH{avPr2^^pBzj+K@O(xg9Ig0}Z zj^P_Cckoh10n1DCcwyff&L7d0u5t+#UL85*tqshn*V9JQrYhs{b&L7fq4nW zAjH9(^T8MV^z}KscksO&KV>?nO&rJmQ9*Q)iYSQWjC$}LCY7mZ{aHNd>vOpE-Ps&G z^Z~x~*=Al$&trLMA(IY&&u7Q?pod&SsY^SKeg12v*Jx?Ak(P#h?pgCs&Kli^ZQD3f zDi%@d(1sJ<+s3u;JjOviyRn0x8>J$Ff;5m*7Vl=I*+y-33cpx5f`RVt>^*xa6AS97 z)$6I(YnXp-9Vf>4Q{n8-`=9-aH|h-bb+s8;SDM8O@rU^4_%3vi2!zO&54`pz55y-h zAt8YmuII2prDOTEgZ%i_dAzSjN49a2Q7jVZ++i?3KVC}f9rl8%B#B=<_W;9!+H>^$ zB|Mo{#%8saTCIx3$;bHq<0BaC?m!36PMrM8YF;l>(b5uZs+^Ntx@bN}-xEbo2MGm1 zLZ6Of_|e8STsZMQ-aFy}PMtoT58mIOVZJUD3j!T{x^Vs%r`crTzujt&yvrzmX9@rz z_T}uAXIQVd{Y$x{x%d)SJTs0RfQDVC>K9AobB8l=-?I3DT~%H zxm-=FeG#c^EaDF@kD#|gOaOEV9mn-oi>Nl3?Qv}}v8gPXUp_UI9#Z@M4vc-6JF}{( z)f;d6og}RmBde~h=KZ1m>@;K^4`w#fZ2j>UEL0Yq=Le6D;-Jy9`O*3#OfM*9Nqz>8 z{jh`+`n0EmSWMTpaa{PtQC4crBrPmV+`+}q&0xQXKsw1J6p5q^jGDqPcYn*r#|>nU z$WXR%kWhd?$C#NsoL0?WOq-T^Chl0t=Vpv#M4&rm5-~+mM@CI~pS!<#i{l3NWMrr> z9V8+`kTGWF``op23*UcX4hMH@PgjMMVv(3$ord$~QO zsA!}{)6A0V2l&$X?sS(+D3!_Sym}D2P4y z;A_kI!84QDukRp^ojj9sXHDmbd!p&>Bq0PbJv-jVFV7Uw*y3l4v4J_)61jWzI~)}1 zKtN2d4#T*3+c732BrxIJ1*R4@+*~(A&%E(dA@la+hP1ak>Lp>9>yw6cFp$zNu z5I^0S$cjc4wOTEkYw~$$&Fj3kiysvVH%8q1D8D_GLru#8Z01TH{pJHcHfcD!1h`Tx z5-4@_nXlqm z*{q|%V4$wKhL?6M;fSvO3>fk%Q>s7-KhTrZ>U{+BXOLDGo$BLIYxT7a!5*hs>hjYoU zYiw?Du&Jq(CpLY`h0{kcBG8?3shA>>l+llT!?Gqj7O~k(%s;!C?<}0k-r@dqlu0NO z$rv0rlV9yz%}2%zWRJ)Yx+(1W$hFrzp2?~EO{|(N&x_D3*Oj&*RFymssAQqDG@;RnBR? zJUonYiG1t0ngGx<}x|unrS8;e)8%iW{ z`bCf9isL!dnr*aME!3;3d35mv`Z-Gph!`^VEnX|t+{^ z78;vNxa))2?Begv_CucHzT|RhTKrX+d7Li}??eZg3!@);i|2})Z}S(eMpmUC;p^=zPf53vnm>>(P-FImCbc8-A_-sgmPzp-uL9^c6?>EP*Mietx5&anDUx|}_{-n3pviPw{x&w3@W8UCIc(7VeENg6Oi0dVSy?fY7pF`(;%F8X-%cd>k(eFOs6Vu1CTRV43#R5fASB`#pHNX965hslq!Lbvj zbKa9ra_0CU?C9f2K@iic(|ufaYV_wvh^rgA`!t_=2ardT9UkcV;MxA82g z&f}JMCNRiJOhBNMPc+}!pGTEePjjpFwb5eIvH1LFyeA-lJ*U0P#3Ch4R@-l^xt`@W zl6YqOaz50vEhP}>6cEJ)pX_PLndf;WEr;3H4{_=9vpAqjAe|LbiUiOpZZl3%@%=egDYeua4Aj==bKBdK+0NaO zUY&+<$=(cV?*OFC&BffmYyo?82w}fTFLP&NE*n)EYPBksBp>1X^G7kfofj1XD2Qe3 zH2rf{sf?}f(V*n@eV=f`u)&=6%o6TBm%^g5GG<>p$uC}=&d%Nr6ib{Kao>FIxLQG@ z1sMctyQ8zN@!cM+CxgUL<<#cTu%X#l_VNR{C zC1=}c(o{0#!Z{`+oZ*&tA7-ehf)Hd38#0G$_MWz5s)PidzmmbSqC8%^be;)kj`FKF zA7%$vfes-(5YnOO7 z;SBeG^#=Qex>A6cKAng0pPS-u`tEGv6&BTMsI184<_2rp@zKK!aTB+2CLOqN-B~u6|M1OGt1RR73un0c`7sQ1l@Nk6V;*>x zKOD8=y>l0@vbfG36UL@eem}oEB?2gMXv-P@T*dbuAIsi-hj9GVS)B9mWDe~aPItMz zzi-=p3OC=03o)yivVS?J3>(OYo_vqH&ZMxYtdv>TPI1+1GZ^LPM2XCaorXNZ&6i4P zvi&g(u5T#dwhta*T!#?$e&AVdPt0Y#N<*zi#hUyie*OA%M*6$b$vu#R=Pc&Q8#O!$y785b7uldO-t55)by?i- z&Rh-{Fp95!zKs{t3s_#3&%^`Y@~P1=bW=#E@C@Zcudd~dS{-dR6YC1k^Q$+W;P4({ zba#?dEC>{b-p}RVtmbPoN3ut^&h&RvPy~U_{@wV`qm?u?=5ga|4>HI_N&%qAqX+-F zD~a_w)6aZwqe)xN1D`y?Fb^M&TC|A;4H}xQMiyS$$EAxN#_IxMq(XXBK5$<*qLmaag1;WfD1kIuGM}TQ5^-w);w1dW}nG zcBi8NLXfg;uX#LNSVx1&($a4mSd)En3!AxMWDJAD`*6XhyLh9#ff}urnx<-AKlByn4vnCz zqZ9qR4C9;MpJJ8zM_*d0swv{N!~~u>d4MZk98E6~C`cR_J^5W8IDYD8uDf_Wk4nq$ zB{HpSEI!8tz1;14TI$4@X&>@*ViK=jz53ffS1$6np` z7#-e@u5z(G9(z2&lXW`M+|0xcZ*ov4e+EVk=Vym9*`U$UV9>Ft>>8Ipc`tpO92wMe z64zZUZ`tows%ta(!~4&2>Ztx~=jlkXAW-Tb&1XJb%}-x_l;cK?;X_kr^T`E|bJBgW z^mdX`5XsqL#0xxh{sK2HS;(;?Msez#`F!sAXZYy!af}W1qC~J`t(P`mzuE8Kn!}rm z6S-vGXbu`ZiywY}nCbbYEX~j0@r_G4sb8p_TMCHfg5`%c>wV;M7t&Vzw>G$W6->|zP$%q$uwm?sdwuVPO zo=Sh2Kme4u1aRWJ2ib53(%GUb=MPU0VMy2e`DJ`5&9wb>B8jDu$3B_Mjvmeo>@ksB zQ*R}6Oe{aQhNHW9Q4CP-)0wm0SjC-t_VP&lHP*CjJgc#W2bVs=*e<>J(szk$G}u}t z@$$q~yg$r?VgZ!<_2Rp`Q>iwSv|4C18mO#D;d>88*hK@ul^wcv=g8TMxH%z<%|@%8 z&^We+6S{d*0I2Zj%C`^ZQhy7$vuNuXzxo+=>lnl5zK&;|-g0Xf#mZ}2IH8N(Uka~C zzOeBsH6|;qs%qXyxx&-?zvIkaeiQ(TWsdANaweaAb}{#yI?Mg*KjOo~`!KfOgZyM) zDph7Z(|5eWC=VGS^5WFbPO$ctL~YjAa?7&==qwXc5i*=#pD*C8E-~vG_``zUbdm~$ zIEYVtca{ye^xZbAnT^F6yp(j2hrfE6J%U^);Fm=bvnrn(7L8+|lM6eKdWk1<8(Nbb zHVai{Nqm1sFS>|8nM*rPd}}Lx2H11T}p@)?0o-oOsM$xt0aH5NMdfzXU4 z7fa4XSCUweb!JU^1*(WANiL=FwNWMP|G&1>7=VGkcEAxP*I zF^X$0RM18%Yl|{@`C>f3d3zc|90dXry7|O%;!_K_`0IT#l&*>Q=c=n zT=OgFqM3ym$-Ef9hi^@50jd%?ePYM(sRb|an|-Hwa^E^WKkYuo#P;V?EB5kgN*XUG zo#)oio?$m{iCuK~C6UBj&jX96F~rS@?fT8+-W%#$@{f@vCzo?rSR0A~D*U^0-aBi! zYtKF&J)O+D7UJeskpzMRL%R0l=-IDv)2R$L>MXQbO{}|doUcuYrIP@P6>c2z!Zz04 znm}k{MbZvF6y-z#68c9?=FTGRo#RJW&wcMaKz}C*A;=gy>_x_xDEVhgk6Ba5EpJa@ zn16eYe{DT;8}zsMz(#d>JRj@fPpJqLq~4tT@d+KBW(xoSAOJ~3K~&cL$i~==%sKQi zM@D$jC8#Ig*m#*utyKbA42?X$?j^?gIZzN}Y(HQwcc)ahkdzi0^m?jFuW-qteS8%kiu0#L5a{67m9K8SLCep8SNi7H+09=@2u_S0 z{}!(_-Eu899V_!Qc;#F?S3N(RUUD%Z2y_V_$VX<+<3}3~G2z5czW?NS_KNMn$uED! z8;w?KD^Bs*0l}0ABKi$|o{9O*Kl#UIW##E5?CI}9k<^C|zP*iQ+FSQY6*oM6FTK2j zIr+T!W|5abf-S16f%No2W2B!QUj zexd9$dNw~;dziVkTH0u%QCZA`pU!5OyMh7)diR^p_?(7YV%0`{`8mEjtskQX&gS;( z)vX#Bo2ilUOQ*7}tBismW$=)vc{0C|7LsOlJ+o7<@M6MIzCWV}9VH?Pw}~Vc))b^O z>EcPQTr`0}4)%Q1#=i$AJ@qs{T6=(JPVD9fPfud+Zm}Hm!Ww2(Yk$K(W|ky<&#@g` zD2Qb2_UN}PR~m0|Uo$U#^C;UY#1uP+^Tn;HRNcyJTg@!K@B<(2*PcP$r*Qq*f>xxW z)u810RkIi%7u)?GIfDlZ8gGJNt4YNR8(wGM_WtZKz!N9(kKxyc{XqCX)QLkE${ba9S6K+eH#{6Z6xqGx5ZBzC6Aw9Rv_k z2m1GTkWW3mi0clX=J8!$^QnnL867*2FMV^0HCiK!FRtV0P-hAv8Dr+HWo7d%{xUQ( zam^eC%EgqpMDXS9X;hnj)`%CK=X=xp&@ZSbU;p7en{F+3Ry_;OeaXT8asmP!y}R-s zhYIcFw#~}=%w3!qFNsshIHj6YSct2VdKpOVgi9s1_RaI_jG8 zxNgx%dWr4Cvsc7G&UyBq+?kNZ8kM1iyes4ni^kF4SxiCd!j7?nIA-cRF5i~G@)rNt zta|31TgP!-JSY|fx<*XluJkIl3MZ3>m$$sd{$09q_D6eIu5Jb9*pPdKb9(yO@82bq zbH6;pCcTBUnrYA*sIJN7*YkSOK`c-ZNf{V5h_jzx!u^R^tk;-mvstNc$l~f}hto$U zP^@rfhyLR^dCFt_V)rFhsf_m6GHaN9^kWVU^`Jx$)4%U*?oVrKl{aiw9n*Hb$DyHa zbZtL?AMQ$_>St?V*3~k8<&%u`lu{5Y7=GVVJepNctBo`_mNE6(WuDl&jN>BRDFBo@ zy0h1WC;9Aa%XsL_8SeP%9~?ip2V)04%ykKQ)Z5Ig&pym~y?rSHD0d6yqie2Ge}}Hp zW;HP7`zIOMsXJd@f1SEN0P`%xr}*^!9VvxCmyljudbIMEI$>gcc@D27p5>Os)7Z|% z-Z#pFdvfB$*?e>5P9~f=#O1HdWWU&M96tLUCRUnhZc65R4|bzMETVhNB<{Fe)5@R9 zt$2PofbUbu}scV*X%y z$X(fc&d0pg>U{S6+n9TtFN}(yqew&tuLwTz`3cq-Zc|p7x9xRy^>-u$1*0dt#iXiV z=of4nUflRPqdi1~#D%dFK4g~B%KwP{ zUv5X5%-ZuF`v%*EVb}J!II|_Th6S?x@%6aVIAteMi4xG1wH`KTL>{gLPBz0z| zUIRJt@prf{sesJ}D{U4%E3WR~{Chi6A+&HIADp1Z+8Ps;C2ir%0TJvoeF?8r>ROA` z4O#r+$pLhifFij&`^@@`8BM?c#w?mDo>=)5yZXD(x7#GHyHvrSd(hOB-%>B66e&E| zf9}V;R-tLtujK51kG#h3c8w~Nnd*i@CSOeC!BtN)+RvFHfFj?{ z96owFUt9JA|TMCi9o?2lncH|V8b0RxRQI9 z%wSaP1a7)o(Gt%l))Z#)O40@HUbTq5{3HZKzaWy>s4cz3cW3lxM8Ai*@oE)Yx6AcB zwtODL+{A=P&fxx!aPN(lJz>!>Ljd5dv=atb0xb{Y8! zFE-v`{jEmk9(aS@JN4$G4OgkQ{{B7Pkad*LjEtnayFbUiw1FiW>un-X@rCa>F~*NF zk(h3w1G#v23ROSzr_E5$Gi&BDKn#kcPVE2eRyOF&e{v7D7D*t8>EzXpT_U3x74>_5 zBH1}CkWOOZ-y)LOOj=%EF`XU`Ka%PXa%)x<|LooB8_#iJpLUc9ARy!5mv^vCY5ha~ zs>?sd+0kzH>j|B}9of~buE%CJu&yAL7ZXqM^CySVyJfh;KdLXs&UlH-Hyz^HlRNnC z{0G@1I*t=wSj&8!nfijGoHrzlarIuE?GXKCw>>KPzv7;|XFWSZ$>rZ^2 zUZ)1$pZb67op*Rt=av6I_f9YBy;oEbMF@dFNc7%pY{$LFu@h(Gth2i* zjNb}J@5OT@;&E#kDL)8%7RiaXKnazx#|*Z*xpqozxIt)a_W>7@(bH4FT<8j{^dD2 zH_Ut9f>M{sFYj*}ieH>^%dtPmYg4@PGej&|M%xD^1khF%inu! zojhYowtV|HUYF~vt}7%-+_Jg;p#1#fE9JaJ_sB22e?;!F-5meqAAWHAg-f9eVCuqK zS&-@PrN|lV=4jDhI8ZZq$v`0)bJb2hC}|;T+F};vM%_@(GG|LEA8swCu}{K3D2~`z zlV$3Ocuul(}IJh&(w zy~jmcRVB@X9uQpEZB9HRDELIBj;P3Blp@$$>#1t$!sfa9H4#K50dbkk%AL*P#rdR1 z2Tri&ZfbV?nP0uVhfYm2i&w8^Mnd3dItW35PvjJCf9OHxCK_-Jw6NpPzvP_*jd%b- zsX?bR5+0L6e7FGt89{4g+RCr;%#&YWdRzcXQA8J%#Hw4@kev{OLUPd2Tt#(#@8@g) zmYeGBf8gLtaz4{hLBQQK>oP&q_m&8*|%fiVrqiK7>)WQ*p2qi_at$t;ogi!nXAc zP?0FOv7Lphw z0v?=_4Zp-yJa*q|3Ky>CE6+U7uU~zIpMCQlQo{}7i%aw+A#)nD=jAiMa2np*xshxx zl2$*)y{qT5aP>WW_xYdm>MOtGoA)ke+SDv&=H@ZKppdk2X}r9bqwxqIY&%SAKLm!x z5gj#>l1NQ-?jjatC!kjfI0u|)!X~j`?OK-2pGL59Y^p7v)D?W~-@ncm7Gx5jQ+Rns zQ)Y9^9jln+FXC}JXgYa<28ZXmc92rz$NYz$<%R!#j%S|xF+cs`6XZqta9zetc2d3V z_x${g?KtD+aQB^SNDLg=8Y`gj&tS>IEc~=;6rz)Xek<-F?qg@iao+gtpV?PyCt>;m zRxO^2|0OM-3XMP6EAQvN6{K5(f7T>~G|MLZY^U81d!BclK$zQcnOp#4e5IjP^~Q$Jzy1=oD_`Yt>}3lk;cK(%fyv>9k_C zx<>OmYr~kgbTL_B20W5{)Agk27th*wwe`_V@+HJ^y!*0VPNh8#^3g4(m{6zuD)=YV2D~=Hyxejv`@BilK zd~^op-+mWMa^gnO*NJK!@wrQxo#=}~RDfdull=mBw}h!`3%~yP?HL)6(WuS3-$|q*#d|f&E=fLqoszZ(4rorbz4ySlaw``yn=ZY%+11I z1l%rUr^Mu2zQkQC=CgR+1APB~U*dPKzRbVgHk)9z3rlMS)$Mjbz-hORmX}hAZ&cKf zyqdi7a$S^{$KK14_kYbRn<|J|cn5bZ&c<&fm5Hb!G=3KIXD6UlhzQs*nH*ku2_n9+ zd3<3_4t^>TkFA028+X!Vv0ru`JiWD)96f_4u8@^E5nLA|0}9ky9Xj72;!~0dR13(m zKtRM)?tSV>))!5X3q85 zkVPdLtp4J4LTMmr&Qc09!%-_Fx*Kab*Vw|y(WdsM5?=fDTNF3Rq|RQ#(t?c9^SJ<} zE{NIdALi~Q8E7R3bq7D-7k}7Ehf_is5Y3eAnanF#L|$e%0^o7EaLIvec>H0O7Z!5M zL;uQ)zxWNWzWf}w%#24XC<%y}#)`G`iO>o-2fI1E`9n_ky2oHMX=3lLG6>D0U|uS^ z8#RJLP!SZBLiX%D7Ot8{yk6yfc9RL8*|+n=gSSvnxPr(2<464FcfaF*zxNepge!5I z8mMaM#_f@DyPPtxkN(7^h zwIJdfHHG!-XAv?EMV7(&Aip~iPeZm?g1)x{e|7N z{YckgaAc2-)0+7t}O~h9tA}F*(PFcvU>lPAi6mZ)H zs6Sas%LswEObryh{X2G4Dw)4w1(U;cqwJ+RAePzrg~SIaksKDPN;a|cMBCNvN7+Sd z)jodr+qXE|qhQ*qyI3(b{4#n!m46~DAAE?Vnc*m%CXW8~b^frm6!S1oo|?Eh>sdD^ z3B5|d(Ne~)!xi*iI-py2(cVzbkqQ?DE9Q|HVjP}r*K_J)Uiss827VsHM=neCNOPw9t&12AU)U{i^zQ@ROySj|K8hJm_MI;{^fi8 z;?>{r^7H@Bl4Nhp=Mz7dJJ-!2P^Ut+x3GWLVOmYL%P^>c_6Ck0t|YQxC9@KPhxW6I z(1Z+fa`PxCT*Q<(&e}Vd6Jxma%POL?SM#NN7Zauz>8n4)?|$_M zPPFuWdU{dzVC(CozP^sSx~uuCr=htMo9D{PNAkF^4ffOB+CW9wDJp83=&@KYn*}qP zIY`24wIazw_ZLMJpJ zTFzF`Y?X)cKCMoNUrZv&QNd^wAY&ku>GHIAawnvlA}E zmXfB-BQGlwh3v-MbdoI}71QYfM5Pz|T9k=S0H?Wv-Fwes9T6`esEJIOK|x+RdffDN zc(E_{SlHL0lk^#rS+rsn{)&tA|EG4o$?G2+#-d4O_1$a82-aNY=Le?F=aGkRAvsuw zz3VLRy!tzKmbc+nY6wZ3!mOe73oxhw8B@25X)A8wz75Nmx9C>>^*QhQ{pjgiiJxHZ zp)5IQuiVGWKm0Ftn$vjl*=Kod?Mwo-;!U#B|Gb8s_Ry3ZmM)#cj`A&ZI-Rss9Aw*$ zBNQafzX&+E%^cjkm8!u2)~#MelF^%(xIyk5-8EHIHMZlFynu#yiDw`fT2^Bz8hLW$9!AIhm{NtwHn{Ny0-GpGzi zrA#4en99#X?DVZlj`Q0Aq3tQzob5JJu-^wZu=I!@{|l9H=`&U(;!}Y%JoI1&@ndT zyE-_I?CDvAeRzU;Qwv9S@8$fubdp9y{%&(Q+e>Qjn{p5HQ~ZYI=rFZXy!iu4JJc+V zG|<=HM6F4<#Bp=r6A*+}XkyUXN8Ond)R{IA8Z3IjB8?HP#_L%42B%WEd=9}wid`2F z&E$aSYuJbi)CN83p(IgAYCg;I5{3pA6=)0FteOlQL*sjTvd=ycj~28Uhw^7K{k-dmeE)2(3U%6xKCFXvuY>4KR5 zz}NX|UjV0F$=tU-n?QpCaBz0t$80}#7Ms$K_>62OjXCp0Ko^`q{_^FdeR!M`y}g{= z{w_Q2Uc)1~LBr0gN~1;z2udvx>A7SlhKw4{E@%UZjts{~3tcuRmfmjqWeB=v`3WeE z#vuSqKp!`oRSPqRk~;!Eq47)&jUNus)Ztm==VftT)x+euIYb1H4#ot9=nc4JiGFh* zc83cXl7l26NfNTHo6_BXWz&IdZqG}?p!N>l6qEtvELuWSr!V3)YcUy*$Ac`n>1jI7 z#y7W;{gss@23{D_DH4z|izT!5_^1^(S4Rp8f@0Ijoso(EsDbGM8vjY8#TpSy!QJ0Y zyV;6c0uvazW1%WY>NC>(_#)68V;BbNh z^axEBw8mhPQj#%VlHd?D;Y5WRP>5hN_t4gB!Y(niFYKJzwV9o#>Tqd8nKXGC34y9H zWv35Ip|2D&TQ{O%T-gh1B%igdTuW4&aa)!=}N z;1EBA>JA)(-IN_JrR%b zl`m1zi3Iy9s2OlE*waCeT?Vz_O=46ZVbhip`i*mm&Up2%=_~$-Yf5+zPO0}HFq#6B#z;cFp{07b4Mtt)wB44 z8HA593{R=lpwnr_vDq?%E$1lO^fC1VKFo^H(A8Lv<&v%EkwpA`eNhSBIQrVDICF;f zfdc$|1@sX~%q`3(dS?yweXSfW+C|;E3?_|;rFAh`V? z0I5J$zdF~}c9iYMx=5JyATy({*1mY?sw-pDrd_n!AU=5}vyyzrd{>3WkF4eE$$M=x zO%*1Zk9^1n#rKjEJ#}~%)6ES7G$bu}h<~{?8-rSLRqtkM+RrOLd7H|9$M~Oe^ih3e z8^3?;_Y@tgrO)a>5_LqT#w-+6DwPNq(tr;8&XqG$T;7z2aQ z3Jna}EYzJoMosVSgoTKx{lb{BcnKNr9N~o7Oli?JD(}l)29%=@i=v+ zR*vl2!};|YB#$_+p1w+U6raO)@|`S5^&3tU4|LbF{euswHoGv+n9iKEkg*G<1RY89 z*RnAE9V$EP={~cAzw9{0g7gT&4Bn(p?o_7$03ZNKL_t)N!JtPWdjB1iKAVL(nL{&6 zMSQ~&m=boyiU%4?ck;^5U!_bvn}7Y*LljPqL~|1h3;vlT8L3c>F%&LY#*`0=*>%p0 z+g#7CqV1Hgn#r7aBLXtErlV}$a~gHhI@T5@a-+xA4Gi>SwR@}n>^2Mi7KzbQkTfJD zC*Y^kAoNMNU2aAuZl#_oN=}@kx4$3Tq4#*^i9MGMwUlKEr^Q5YL<~WIPGP6V?(!-+ z;X>>p3@avuc=-+#L=mM{i%I}lmT}k}!^tAIy@`^&rS#ftI1m1X|N2HTTJiFs?hb1& z0WonH5it6>F?G2hZ6tOl3}<$IgF-LA$8e@q%cLb=<~IpT=@kM=Oo+mu9CFm`{WPCF z$?>!8I6X4RZd^{!byam;2egoFf)ThXoxjAiPyDTKX$ zjM{z+b!QGy(eNP2sk-6cx@ft?VdF3w0MejVdYv%0+l|ZZ`dpnkTkS~>RWvZ*7Kri- zA~@K8e0c@MF>TspLN^|#e!xQQ>BCet-cM4B4irWvO(|f`wBK`PcN1+Dd)RsWadMW& z4c||8yRh5rxMcy^*~i%vdpUmgP7F;$A78{2L5xv4jzu65#K)EntEtP6N0)w=l40sZUU}RB5?H5GQ zl``o9S8!lt`Yruf9ZqCKY*q^c1D84u8WJZZ;GGLX?8M``w9@0+&!tLh_=NgcruXzg_MNH6ruHQ|IgJR-7y=Vjgk~(|n?EhUC5fmt zYA?9xXs7hpNt(KBi287XBcd*LB3*V50|}|wWF!Vq+S5&6`#FvrJIg(}bI^{E-31LG zDimC<{*-9cYHzYzl5jhnxa3c`mjnT&Zy=$1<@kulusp)+*aT*%0{+%7zlH0*e66J?dP!Kum zP9FA%Xs#8Y5Y)t`q!1Nkq_*}v8(;b^j^0Kdxoa(RrY7R2RiN-m=gx;GA*$6kJ&`Pm zim`RJsKIE|c=dtg#A?3~WJq+JI>w3f?F^b_cK_jr)Na1~?8uS}v)w>+Y#ay*#KB&i z9ybi{egT9lrI?27nBXmnP^tY02-IAkOav7XS!?;ht7$Y3XoyRU!*9rImEBf4&R1}< zv<8dE%Ru0AI*~5)fw;RVDcVO(kCUL(7-FJA$8<^+`fwIJ@CxvX?FlymZAEta9gh*w@0*4{%yaS{7#O(=pgNr{QXI1Xri5#$Rmn&1?_$HO`-$^9PRK43Ri_S8ZA<2B^HPVO z?~@5=g#8i5_Mrr=2F(ztgdiXa;utwyZz3GQ8>IL5JBR%_$<> z4tIo%aDZ{!%@`tMh?7C<52h~XMgLduo{Ni*I`+G(RBBWz3ux%HH z0(T|;!J(+HkR%PCGMDA^ve{R;gPyjt?A%(!hQeer0)|eYY~}QUgEZKKxqbOuf>bvt z5*0^yMh1U^4eIfIK%X=o{QgVKd7vAg=wxEUh7ur>n}Oa|%8!)MVzDCu9*-OMh?a79 zS0#H(N@=&q1SiFim=rkf?+D`9Np#ucr0(E8PIV4q*Oc<=fBy#`=r7f^GLp-V$rwdk zoQ$9g#BS=sH9|vQtv}LNd6YxdZ8&8G{z0Jx`1_2jqcxFf6w~uNxbf) z79Z8$XehKGpi+>;wF{m?4~K*riz&;DKIMCCq4R?@{{ zE~&SggU8R(W9et$=m$Le#C~rA_A)G`i=c>T0%XuC9rW0}_UJ;weHc^G842>yGocz4 zlmx}iX3erJwpQ-J+F8oxqB3sFjv{0z9(50zDBr&uQ_L*pOio0vxW-sdcG2Ea!O_xs ztTsJ+-}o5Cn|f9 z1TDUiF@&qdapmb3Hii7`DE?Am!Zp~+p#!B@Zpp-;Qr-OHC8`KX$>)x{ZzVu`H81F* z<&>Qd|L8=r4(V5n$>U87V z!C@~{Z72abuMqA&Dd`1kWV;<&ngHgbh_b! zG(i*@%MDa2m1xu|?{B#W=^v1Wu_a4yH3v&7ut^S1{`Kd4qvZGFU`w`PY{{q^!j?kR zpfHd$rGUB9-=J(~BW;y?*?GK!nJeOlb(!pTj>5jqo!HB<%DYL-zJO1H$F4vAhU%bIJol|HGc(Tbx(j0enIySRam3DC%<{Z9Id=XKW~YOi zl5K21emhg6r(+cQIlX@;$Ig40_3hPU1!->h6Bvy~v>G*nJcy~gfyTx@wq6N62S)OU2_RQZryuz{y;pU#z3vqBv}O1SU1j>r|_-pkdzxtfal zKHT__Hg7GD-aePWE9gyy#z=gUe%$+CDLXByeMVKwxoy3iJ5kJ@14pTAGt*k#H3ZAZ z+&rQciPrjhn!9>&%L1aPLOGF>E&384pFps`n%aH`y`7CTHgzMV##|Fs(8yHE=ZkAg zqNT2$mTnUsSwIw(SCm&mf>&Pk0}gsR8fa?nK}v~6M8GFDgTnloY&_UVTUQ zMR7!`yuW3xKSj~*lL)^4XdHcX)s#_k zXPa!Gh+yt*_p^RlJlZSlNwsen>A~W-N9KlBTnNGq0DT2~e0hOi)I!@UD9(D6V4DsW#3Z{(v7fPS;;?rL@6`LoLjoY{fs3J`y#AC1nvd z0w8IuI?A5i#gsR6(%w*uRpK%WEDh9hvZ@`MOF^X352JB{6Q@+8SE@&Why%4%RJQlx zQm8Tb1q>BOjN{#@btpfrrN5Fs7PX_~<#7(sTzQP$dy1)O=%B5>mJ6&PBmEWG(@*)y zDoi#P8iNrZqj6mSP7sx7$4Pdo{9`Fxv4*Lej;dWmv(D8J&QggnAewU0Wh^WRa zCb)Q~z=&w#B12J~HsR_wbH26~vy?sN`w3U!4~T|%maSXHpEe!gK)r>Ndy6POy_QMy z;!z18nQPd8WPR{Tx1dmY%)_+WKrG z45B=&D|*h?bFQHYyR1N~4FfWq}}GCt+}4| z&OYRr&rfIhjn9#|>8`5e{GbalAd^*hJxW3PmE@s79}rE1;vZFNBLGQ303?tv`sb3v zMtfBy?Jm%!%;dgD?;tV6aD|(x2#QO^_lhF*U>m6BOjQpaK&@1vo(ObRDDX+hWRky% zvR)f5Q!N$s{kW%tw~j)@7?Q@aHTnFx_)R+cYuNT-36Ct8M4XQrBsX0ZJ1J>3u;9UY z#E#&5vbHo)-`<5oX(TKER-U?R#+9)Yl^>HL{l~E-SGS*3#t?Fstzp`m#T@9caBAC! zoO)ydi(`#|M1NN~yGvThT=X!hF~(~@+a4RHwkpndxKZe0S$f~2EJ_Hz0vITu3yvq+ z=ORcQl9|VX87b_mDxv5684eVmV9oT!L&**|9jErN^LPt}AU`Y}7S5k2;c!(0IdK{2 zM35~tY}<7bF?lVErvzU0d&mwO-PPqZ3`(F7P^#1ucIatzzD&wU#webl&*P%Ks)A;x zn+SvI(>X|@h)Ss(DyX>9%YjNcQJ1@frR5mEf9aQ$N|Sl&2jAu1rPB$~gSokaqPJe+ zhcCX(rr-XQ*yL2cbI&Y%6(h~3h)-NR;VK2~xNtjLsHrjIo+V#_a(3*8?XRn&Y0z7g zfHIia7=JWs@zYXJBa$X#`wD)Mlb8{C;c=aG)t;t!|9(zXG+}Bfr)$u45f{Af!s#oY zY+?9WWn|fn+wB>~maJV>oNcs$B8)k=-p?&F60SgNDxmfWAtU(WS@elXr(oVJ-Y;pO zwY!e}+xPR(;`oc$*ZDGb>@I~1*jHT{#YaxDeCmAs6(C#NIIwp=J%PC_%Za*(eRUlcPCvoCz7ndFFJ$5P#fg}MoA~%p%{9#;x%SJ0rLT`cn>Rb{a@lY=N}!Oz)D zkx#T!=<73M9hR3Jr*mR?g#-uVW4I`pYIzOyc81a9{vhz~OOW z>$iA;a!8l=6Zlx-h=#KFoQ0@HGsxAGohHAku0dVu;q`xyS6g2)-`dFI(Z z#2>uKrqUK{z4h#R^XF6?J;3%`@8`>p-p2I!Ag-qyUg4X}`X`^I&jBy~aW^d%EA_{V z_*rcQ`**G2{>Q$=Z40Mfx$Ek0wEuc~&Db0=qK}S*sS8-WEcEjOaLE!*dmnWt_OfZq zPENPFNy^D*`4^TWHXAs#^(bAJ*k4a?CvAOJ+_H=$U2BJ@OmAl=7ONduRv~#jpD)`f zdvMsi^4h#*C#TyyB+F6J;Gj0w0{-;egp^j zp$4387uJD(Y@f(pD2T{e!MgmnDQ!85rT!S3w;y78b}TVQnU1q%oH*OZ)cfa?BHN>5WIyElKx&SU4%#ZM)F{sz)R8|XQBgsRSgOTOWvtFw*H?g3;}Lj_FNb7W1l zcMal^WR%7*vh$a+G*x%?9}QA z;c(!ZuvJt+SXc;pjRtUGw^^~-Y}d=A^7rA=$sW2oI&pX;WTg+2X3S&dqDlV*K0Zm* z=5o;4-h~?(jW&eog-gg!3c4|Ft1RQT_R!tu?YEO1Hmt6R?7DzT7ez#<3ZV{U&met$ zPUI1#@d~XU83k*Zm$Hd>%Ll0|{*Zljw{q{)AS6c*hqe`CjmT#HoP-g@05TS{8Oy*R zvY^E$GJ{2{*WmMM93+L7*vtZ! zC{UB#ZU#F$Fu7$!y^gpU^I4gfaLtr8f-#=OOXu?1$EBR^Y~k>M{Z!vJkK8B?(qJ1U zJBz4SP3O@^wEX$!?{T*44BPjeF@3tij^en zwh=Qn#(GY*8sEqW`~~QfWL%a$`aGXgltpr5syW1gS~W`_`wI81%pqhbmI;WR%DPAX zjqY=2_|~7!u(N0vcP-6kTKEXriVBP|Q%H}{aMEnYdgWuB7*TC3_Fnle*#&Qi3mbtICpuwYz7C1ppjY8?Fgr4nd(n zA00`sQlts+^jk1H-8UdE6HuyTJcBNr1Kk{Y?`3|Ok-)bfT1bfI<|^fX4sO5fQW2NG zhLzc|-Xw@@qk8vWDJrYu-02dwAL?S}s^w%w`QH@bsUvc@=fS&}lNOF*u#s&W-=gT~ z`61pw3A?3%U3-qvD+Q9j@=k7}SUOticK;3ZxXXo0l0ml8)!9Y= zfam7x9FLvW^1b}*+fVT1_g4l=eu3w?e4SCp4}h-yWlP@qyON61S_#Ef~&pPGn4;ik2^ zl!Heruu3umoi*&*vL9pa3fA252&=Or(TH{`4<4YjwhgD#!nxADoE%JGNnRTMBSBOd zqzhC=l8Kh)zAGeiKEIbbCXf3czLVVKP((`|AHDYuyGxs}xn+UY|2xb1- zJ6V;NfadRJ!+LPJ+;}|TaoRC;cjLI`gFn>9>~dh~>0v;g$g$C&(_i4e7f~uzC_g8% zwDr`nf@yV8!rab1Tt;aqx|UqUf`)a3P=bx zAWC*x&mQ8fpFhi!Pko<_M;fkbc;V$j6$y-=$)n%?A6|I&OB7^AV^D$J)X1UDZ}99> zPx9PvHq&5sdE4gGayiBldR%bL->(%2yw} zo%uN#g!ztLB_TVVI6WR@+zeRE^jn=m}-R z<4^PCL$^_wlTMh=ICOBb6Q|SLIpOHHVD9g~?meOo$zbJ*1tbM)k*pmY-o1q*=S+B< zR_c%J=DZ<|xid2Hza(2JOCCHf2Tq3*hr@CCpVNuc<+^@K(@!E-x7$0oRCZwQ?ZjmM zET5o@T@#Yij@i_Ubs~GA)uPvGymP9=8)Gk&#za=$wt}PpJvh23-tsZW>$-3|%~YN| z#CbW5WwR4M75?_CyrZ}>^-J=Y6d8betTJxdjom(g z-6;VQE|(RX?bIBxG5}V(OvSd~HL3Wm&@GcH?x8eRiBqr+5C)1^SNRjv$Ev zi-~~&>xBNX(V$1EQXn9pP$)0XV*E$FWErQ^OZ_QZdTBNFVgKhe`%f0%%Mwnfx3bvN z-A40(og0cdkD!^BoV|2*J0_s?6;LRB@EMw6grG#N7Drc@3o62svRSks2S0^GUuzj# zi%v2qfwlP{+fSHC$zQ?LQJrNnZnqncM?!Wx>F;Qv)AeZ?Gyz>$GV>NLAl^@by|0yn zTZ=e1AmO$)v3o};nv{IzO});yFbTFTa&_x8BRbbbkbog_FB>Q`z2!+t$I+{d;Kh$!1wjEE>hNNarH$ zk}ONux;yE0PjILe3e-NvVVq10D!*lxnmi9?PXrzK%jxM{0CNy(8j3`#PCU3Ki*ychk<@uH>zvP+1tRmcA3*eEARTI`%F<{Juc@ zJ*!9yg4#nLv+?Kv`FDSvuYTnL(gR2HM`*QL)M^#DZFJOEP}AWgKKcqA6S51d)q_T_ zxr}r3dM?pmK(A0B$Tk{lYG~{1Co^c`q>?OoaM&Fvv{&Hl9P`T_T2Fq&kDvW7{#e|@ zf+v5(x4(7^Nx|MhobsmzoG9@3_s5`D12zWCy|lHp;U;KeE@Bx##OND9aB#rieUe0$ zowU`Q#qJY_zrR0voyIFKQ#Y+`?YIe?P+mjK+ZcTV3A#jHqF*}8mge%tt_o_p>)2O( znERHc<2k;W?bTYAd}k3ENl8d6=J3jnY8uZUW!u3j7Dk41XxBju({5v0Qsn3=c0oX4 zFrX6zkj+$`I)~YsPq1#{nIe+gg~RPZtD(6-K4hFxQ zf*#it!me<{Wal7ONrPUi{2SR08B4*ykU=X6cj|~v<7;u$bYv`C#iA)2Ik&5tma<~D z7nhO~6vTl&6@+Hp%hZ(M%ltN#A&_NXex8I4wsB6ofZ7m5O2ns6YbWS*7?er`9JDvq z(%5Wbc0|a;V3I83uvrnb2GpOTI#Bu;3GnqnDS*>9Ku2>ET~6@TOynne9UxJwCpat& z-%t3wf1{QB9b!m2f(o$f6RpY63q+5K!tu3Gq`Rbb{pS zptjnCd%=YvALA%3jnw_jOZvmm*xf)elg5hu#ohP2RYQz#fhCC zbK>4ybbIztEyeMrrPCR`C4^B}N@slq=h~el#cHmMr3|{!7_^@ZOHukUeeMEgq-~`3 za4R(@cCx?pUV=1x*jFtv?eV1~>96ztq5_rPfEKX0x~Vu>!{EAf^jB!D^Ee$iWf6^9 zHB7yz4bEoe(kZ-myot`Hlk6xt!@6mHoGsbR;l3oES~81CiCPvf$>HrIo9L_B&$i=@ zWVw3TyRVYCdC!m$?sskU;z}hx!9f^BFi8@F&2`ibxJmWZOboFAh$6)8w%r z;uABI2fvzzM}7IQdO;LXYc(jqI}7r%4jTcjUl_TImyo*gG^fojdd?qW)7DcgOf0CWJKtf=)Nr&) zkf`LGbC?DT3Aq|<$>l+0H2$)0n3`xjeiuGq>;FD8-@^(~gBhuk#k zt*I>*{VQDs@aM+`#fIFL$Zr9OT4@ zzvb7P&XGEQDLJW8+|0r%r7novHTQG>Erm?a&1Xi8NX5RbY%VIMO_j<6-~18JKl>$S zCtS*-EaDp&M1WC`C_8AWJ;9-)wM;w@S*G{wF5cRBkZ${pX**E{h7;zaMieBv$`5m( zyq-b#1P>ugm>Z7q(FZ$dypHNF*FX&){NhF4C^~~9Yz7-v6_7BLAo+AJk?_PsVnYH^ z0xnYrXKU){b&W6k{{DV!4yTvXJ!%rk3I2cgPPwhQnBV@-|FY>!KM_eu#D;ihKe&23 zsH&~QG=aP`iRl+*X@Juo|A|e< zT5$i3C?;Q zqf?G$)txJu66JqY@)8sQ#Kr`nQVPh{W{US6rK!g@!Q;s|E$2D3?>L>#%LSteoxyFl z*VS607c5PP|cp+tISS>5WK_I^LK5CSBO!+GzVdvCAhwY*rZ-QE9C_0&^UPh~cV zjGn!Og)>I=CP;oFD#6>^4=+zO;6U3^LuqN_&+ft!<+i89s5wHSf;sDI@Sw$N(APqee2=+o|VmX{w>1wh@D4VB4te?#5!X zgUp?X=n;et`u}2*#6f3eDxdxP6Vgk&2pklGn?eR+C+Vk?sA$z)6Vr)SJ^6|IIh0(9 zMdJT|U1?*LYJx(7Pzzu;*O7SSJgvrSS0dW9^_)4dmyGiEpSW`}Ts@+R9p#)+Av*LF zW#^%_+6J`W4qF%6Zl|P+Yj`XZhIn0FXeqcbV%j`r#YN(Rnc9Lx&Ky0)k)#4V$Hp^n z*p>H0z|+?kUr%>rl9lGF3#6Pc!}Jqeil*!gKkP}S)nxzGcG-L6G?vba!`B5gH3ghK zbCjb;l4$W9$I^MxzqrC#P$=;T3c*h;;INq~NjO4otLZ0577e*4_jCM$lN9eg3n~xB zFS(JiK`!9vAobX3vJ2BWb|8&_8S%^*GXyseKjP*uWx^m8Hd7sE_8%oB^AxEqfh?RA zhrh?qOjN7g2pSPfgog|y2Zr_{@=69)?2zns%o;5Q$;s_SPnf_U)&F9_Jwb-r!yC7L zq}CiZ9VNLXXdEs?j~GcnphrIez9*w*-N^FUQMd?T)iiKo>n2VXwqK_|6w%e^vVH3= z^6PX+f|8I?Q(3WOHiO*$L%fZsBk#y2KK*(>6ynko@ZnFoLNK$Ucr6I6?hFF%Lrd**_y4pjO>E;fn!Zc!rIkE zw_OB5NyON3gnPLCW{C#1R|Gdb_5u&wFbX%xO8J>@`QVeS6t!Fa=eFoShkMacl6sgu zsjUPB_~Z6l*f7a~rt}Oy$#{ZDxR-FNxN-~Y;=|NLkE{9i9{|B9J} zdG)Um>>3`)sIWj>WD=&PBKB_IPIm18M2jS1YRTh=4O_|63aI73jpI;yjb!Y|0F-h8 zM^_O$x9lLhu4@1TEMn>`<-q2RoT;|q{R=cUjpgS!m{f|vF5%`MM3A@VRWbx(-+tYG zwSX@HY@5Bm4E*)ITqDObdweurDgnvdN_OT2YTGOWF|0H+G@qVNbLcKQOw1pH52 zNeR2nip~E2yk@&->m+H{kDM--5fSZ+Yt#hhjEllkB_Nqw$j;2BuKil;)#zL=&p={l z%xAR!)$8RNGMSqe#^9!up)Wtr)(vafnr&fn{3^zUs6bHQH+US27slZxlW5J{$LC*e zrNJwP+0$e2xbl%QIUa#AjEQtd0Q7aqZ2tBLwfcd+r=7OSG`9b^g9?M{XSEXlC%kNi zX3p)}OhVgGUVMKo@4fpk{_*#>`R_mfnKxd0j(e6(W01#hBn$Ub2lC=ja?y_cnu%5k zhuwzN_G`|#lE~39LkXB0)je-NmC~?G}skE zve8^sKu$q51BH*4&JG%zTQLa=f}^K1cWT&g?vO9Z4mf8%IDRFOF4-G6d*mDy9VSqD z5fU7L`;|7^Azso5zT)FVb2TTv-$8nlu2+snkh?HAb^(8Q?H%5FVkHsY3T%2U9c3j0 znrX>y$KKDqt3JP<9Vhb8S;XGIaPb?;g1esP?|*xZRa3%Hh&Hr!`II#Myd+8QueA5b zN&ZI1zW4tH6;UHbFv!;(L9$Srb%O2t5@;D9GM5|{8q)W(en&QXr4rcyB)jxKYV;+{ zx?+wWNTS8$K;|AuSfD3Y#Ui5E@4b|{MKO88P~25ABy%&TcWmWoR&D>1D3XY=C6DjF z+CfIM8A*`g78S#UhyYYFFgNG3W9u$1)OPnvf=Lb=ExAYdYDX%~X7N%Rxp1i+++lb0 z>-$BA6}$bnKz9;mT@l9*B-3IRk$D95vtAO?0P7{l+!!-?JV9=11j$U_u!kO>CVsB$cM9K8AU-hyAaUQk1zzGj2>U zYNdeKUB!W|+es;F!#==XSh{ODyy*)L78~$V|KAOV1odF1&6r4-zZyZZQD2fverY|n zfxc^7D~)X(m}P20M@(nd#6iD_zYtXJ44-uiH_sS?QnJ&UyN^%T|3HoVXr_%Dh3Bsd z3wmTxL(!>t^)c4$h^r34vS!DXb^HFOda*4ojXVU zK(@8-?Yq?QiA?Unm~o@=QYk>}CUf5p>`AS_FaRg&uxQCS^bOx0E5{Lzg>*+3m;@Rw2^Ui zE9dILH7EqXG2BIkng@bN~g;Emls3!Dn?1zZT4&W zV98ZiL3h5~h|R+RYT7`j#IPDzOgm{Y6U``Db^t^++xj z`A-pM0U9nbhcmtV1gE(Cvgya{32(uVs;ctL82q?D4V|el4JE=EM68*q^|;G zYQjcLW8RG6C><6mPk+znn~%WASxg!=-+w$gbkm@)Yt%Ih#0kXq@Bw~Hvq9L zT1>Q6R?{hhYw!f(=Z(Ns_5Z?21EP(_3kUe_XaUO5ajaaqoM8c~eraE?u<_je(1T2i z@BajLE7d}0D6 z%?4yD7Xm}WxU$|ylB8eE^Zj-pP=QBa3^7s88Mub}6gF=;OpVSwAh*=q_TCxT7JA>Vp9A{CJb_NU%rFJ62B-8cbPE!&5&-mz@<1}5P8$gmoY*s5a$ItJ+_zf+Egd~ZVkyut$SPeYc#=f)O?hnJ@Hs~dm5iG-1VtajBRVYT^=6Dl z17?f$sz=IQd>K1;B@4#-BL@V92g4^WWYLTWuIBM2B*}`cFNx=HVE-le&#otH?`M4Q zxT33eShhrVIW#+~HVbz<#9kV$qk z?NwQP`_aGHUmwi!MR9ob`C+0cULGj)1BxZbr9i&#dll*+W-p$_AP*HtX3EZd%j(Z} zl3Uw>rO%>>b_<$@0=BJrmmjhf%)fa){!02iKi4`-mX<~u^%g|H*x5$2M&GAORKjNL zqPnpYyC?x7c87EYXccf#tDJx;OHy7{ztydxsfoPY92z?%29KW0%9S%Om)p4>BQA_gY^9;QExeiLT4-=OyCDu=kWNF~!o>OT2rgGEL zI4;K2O0OU$%wNnHe!&x#pi@KVY!Hx_d$A(^`Fej-)L zN_|x^`K1jA-b0yp>zzy+>~c9V^OJk(T<2aJ(NAl&QmI5PlL2;2CLKoOrG2s6td}Z6 zq|1HmKu|)mwb9V1b^1Dsj^^ezjD0G#C5MHM>UuhRhP62ySCGz(lCYbbsBJJheS)E#n&$4y?w@2e&|cd_chA2$Z1zhsb8;`HEL+N$ z5HDncgSLW0tbTV5M>5LM7_7a&S+tw!sLtZsHLE#ejAr4KAt+=Z_l{xN(rE;HslcHp z|Ik_T8gysxKU7%|}}>Mo(wTkV_H|a!)Tj6mkS4x*Kb#?bKi9BSmu; zwIvypx7vFW7#7TC>FP|^Wp`+xyDMQgHBr}SbgJzc+Nf#jzAWh`TKcS)B;l~xd+(4S zlM^y!84Je*p_GBCHJ>g2{ukdI&7@gl?CoxpL_7M9YEEuh!&eC=CNG|XUw=BLf()%*tBINXSWbKHJ&MhT=?x?()IZ~sn0o-qyaH#U)y;3J@;qEqInGQc14h^ z)SlnL>b2jJUfzPK4+$hWY;-r5aNvvg*^p>v)~a|0_tE#3r0YO6Qr}x7UL!6VFiXPl z`O8^0c^K{rFxKXAJTZeNegFQVy0nnusumP}(JZ+2P9_E^2PBlG>%aqj@r_s5SeD;< zBg1_ZSai)~CRZ?c!ffKAy?=R-s8FC($dL&WcAEvA?otK6=&)h4+WWaH65>FJ(xIWL zxdW3~M6#LaXl+K*x2##Tp{;ME-Rz77MA2~>K6j0XVa}8|0$gNBmUhk`-_4Pf3d~m+ zX~|)wyR`#@==>fwlM!pra|F4P@EMDk7Vd>ikm#;S=F8Qea3r&mZgZamf#|^0-Avlv z&w2mr6#VDh!jzDof1i|IBe?1Ir9}C=BS=Q_P9G<=qM81hqa3;_GIC1kGOCE0u#g+$ zCtO8(IPsR$i@yfGk4}tu(U(cRer*8aOjAqJ{%`qW!w)1BG-7uAxy#=IF-*`dkPFT!5 zX2$vQP} zz=5VNm4nBNF6&_EOW@a>Po}WTj_1(n+_rosgM9{&d6aoFK0cmF5dkRdUF4*vqY0YK z!q~z6$}9@rBbYmT0wLZi1iPO6W8blM!*+6NItc2@eabkSIREnN7d;m=A)cj^B5;-4sLf8JxI37I3uE#6DUx535JhQ#*h)!ocnmXV#NsZOuxM*J zym2+3?@XnsyD!d^u$eSeq#xqb4>!^rGLsqOBCaT$5ODDs%#3A=80Id+sx9a6o@7)b z7O-UY#d(ziuE9}EpF5jy59n?wBBjvAqEHpOzSpCDc4BL={$($#p_ss0niocWXRQ3%*Aot)pXjx}3OUd=684Kx-d z^3?|)k)j_;{DQcv_E+KJ!^CCFi1l|tvNUpP?@2m+r*QMKv3q(WyEx>$PZ&4njTT{RfK{10Buv6lI=b%V+D@k=;LvPS(GwRS7orA*qQ}~b^R=PV|s4l+1;jLe?{=4ItVjg31xOboT zi%b=YT%P<86a)>QMEv5}9Lw8>)^iB6R@_94ucF`IVly<;c5z6g zBw^LI(P8=dSu0Cz5?_7!J!dX-(G|L#EtC=(=!d(j3Z+7ULa9WlP@r=0z{k&@z(9XI z`j)B-o=jeGJB#-xvH4sjW^EO_*Stk%Z4r0gIFDh$p2$QC9W{9z*|VF3N;MC@^$Jsm z`d$Jrl8Di0?zz;WbCBSbL72mVxx2fk{ZztkvGf9WnbMWv3vOrCw3B>#tQ2#19f#Jx zO=o>6_pO}Ih`~N6L<=1aC7jyzBPT0Vta|Akmh`mPUIb`ni}lj?FS#5Zl1U&gSt=l zQ`L~pwyg)56BEXSP>)O1oT80}>?97HNXHy9kq4gqBg;o$K3~ITvtsF81AD(!;G)5R z-eSc8hznCb^f;H3~hE_=KtX>K7Rdo4EOt`df^V*+d44Y?7hLe zrbCOxE&~1EbE{EHYeN+|i3j=o(=Rw%=3wOmlL%BP0D*|Pck{&E8Pu;kNOgTGTYosj zoYA3-8{Bujtkh?p#;X>qXDDi z+Oi@xn-#OUXW*jUh_Roj!%}sc^y>J;kX68&fIf$h0ES$u^2I#Y|g)Mm@r*Y4ejRPiJy-bGP*Hn z>&d@RiZ*s4YJEM~X*n?B2Bt@OAlbAefA=?@`uh%Q3^GP9dy(~PUu4oCkDpEO$Xp{? ze(&8JN-5@OVJn8_Y`*&2OO)r_&Z@;Ti41VVVb)NZdYrw-FAxy_EK8>h$E}BM2;BQ5m@Jwq5)SVryV-@Qw?D>{cTRQ6boC)wtVR>o z9-Po`g}Y=%Z!kK~lqBM?S+P0#dUXL`e?L53T@WNK?e%38mDDj~P&nqsVzP>w37a^N z5dk2Y%~*R9XrfV%p{MKYs{1AP#LriaTrMD4n@Bvcm&22zi60w+Way-(D2;=A5@|Fz z?Uu2liMp0nB3-puM0W;xfW_EE(!K-Sxgd)1!EToUGP#QjDun_!zYxMF#9bEHd3Xij z4!Lwxr*rcQMmTeTohzVT+}KQN*6+6 zW)K&4oi_=<&|d}h7Od^XWaOp2Wl$Ya(=M8X0Kwhe-QC?SxVvoJVdDf3E*p2Z z5ZpDmySuY-cRjrCcW&Lfx6Z#aH8nL-wWOzKtzJFd&%?zGh=UF+4lV3c5d}|}%IDHq z)Zut50n6uY`)u9y%&{n@qYMDfl=KvEk_8=9%xG~heYMBg8QiJL5*k4<_dB~jmy%rb zkPp>n3)tV*)VSq!XzP?}Oc$Ma15>V=R>90mVoo^KO}GeJUV&zyjbCrHl7mjegfas6 zL)3vElT;SpLzp@@6tk?u@`52Y&yDPx@oMtUQ`^#Mxcw6ZURbBYQBg_PqvzS{8%V7P zhSuMHU<1jJ)_i{NFe%QM%)|rtK64^zcVqZ8S;;uRDTM@84wxvOFN=jzMwPr{P7Zxp zB%GPaa^l*U91okg*)LO(Z}p0)_<&O=T3%~#CSmzBAMhn{BUP8`z~9rWGahwc#lJBk zQlWpJ3zB5>s)lx<+u!XSYZk`ClcI@duDE4eeZZ%m_G^ zrGQ?TZT9e9v2izZUy-u3FT8!EBz);{?zd%uD$5zSX`RpVz%cPDkY<&3?6R{Sq0-4dO0xin$iK9 z+DG9SlGgln`B}tXm5s$_b%tAfSGhez2J=fcjprY?WZQ;Vee`N%5dL|3H)>sW~I0jX)>~ic9u;XVTb{IZe99FlF|1)RDe? zE=X0qX{Znha!Y3;wI2k#(VCS0>2EZZ4hIwg{uUCp)89LEzjXD>8d7dLlEIbW=Ut2A zz~*$zXYfTbFB)!y6W`pD<^m*B^4fwDUh&g!dw*cIQV&h^Y~l7+zy!TK62{3(-a@zq zjp$w7I|C;KtF4D*xQ9B)K+UG^=&*I6q1X5^7ku*O8H6%%%r<xbn>Pfp7&%GWx=qt+%(L_3n(<=}2KvQ<~Rc5m0V>2Z!W!v^CIE9FdA&s{9_!m6+12b-;)9rer?sRJW2 zM@JwFxD93^G_C;)Jv`!{Fby(P9?VY5c6}FeTPMNJ^6MO@Z?|^5Cy^~D#dz3+JK=iu zQ`(yUY6RMN^eJ~cxpK_C<|qv5wCS}NF+n;E*z}zEi?~(J{c`&#{RtW1o4at6k@|09 z00t8avn?fe?qypIc}g1xTk~ep1vO}iy2pcz<&>YYve7Xy39}V`yO8B@=YakSmhWEW zD1X=0%(BccDtp->$95+lHl)>ruzvpZVzuodwTOL-nc34IPZ1f@Zb{I}5!6fTNeQ0? zpkXy+lxHgX^qQt}0lsZ?Ro@98@TN)t`TMetLMnt3r4%M$+|PJMmxSm^n`7al&1Qc( zTH}Q=x}tGDX=HnOKq|j$j#apDS0CP`dP{qN3_i!2w=N2io$47CkBEkkN2(XPwdS)H zEYI`qet7pT&W=BhNT$mLOfxf9Mt8{tYVLE+hc{hQdfHO(DX#L#Tr_A;X*Oyytf1enb}zUni%1&ta>cq*5kji}T-b&n zVb;O?84vB7XJaz6#^UD}P=+y@0miwx;~vK{4$^$pME zI_88rPBk~{)79X5)Q{E_nQ4;dhv6x>^SrT0!;wb;nnXwjv;F*O%^C|` zX`|!uX7?njnipYKW3EZ!waK&Ato&Zv9!pk}g(m0oum|pkAo!#|c<1(&rSr$xA9{`t z*QP%`(02j-E`w*I;RrWJkVpp8T$gijvC8Z3Bvf!ZEH>pl&*R>i%kPk1-wlQcDA{od z7>z-md-IDPM#lIX<$_`59jT`Y78Qsn;$-uXA?#LQl*5CCwy=R19cR|a>HF{P>DTzz z>8{@;UPy=sseEw#J&tfjbl-=!GbQO)peEig9VRZ;e|2>s$%f%XOGeUL(j#@f;wn{S z{?d#Er;y`~tMe3M&GAJitczNW$T**vrV$c2ViW=i1S2JOdxt6(jp`emUq}6?;(<}p zLII7%i0z@176%E-dP9FoL;<9LO+fp+JMKpu%G%praQHG4=RFacv76AF*Wy4LyCnn-e>UeM;zb8Gldn(eOvUDqy#F zT&lSm?6QI24DOe5FrTR=Mn{B5!jW38zGqWuK;gf=&fs@BDI6^qBX5fjJtn11hZl%x zORCM6y#-OujE3<ZKCG-jL9a?ZX$Dw@u_klFaXUU>1G`EW)74|gS5lxT>q_0PV+h}C^o#ZW@V zkMjt}liix3Idsf)CDCzC09s?8CphEdP_N40fk$#7d(wkzDGQ5GyT9-7v)uw1NB?QJ z%gkLo|LbuEzwe1SKeTwnxAS+DG5y+K)dd)bQcMRE=B8EHGxS?HoDn>1CDn8-kegRm z^ywbaU#kMs^FW%{r)&~)A9r-Mcx}^MnALB)i#+wiSVBZf%)r?)mTe(-p2?1PR^VjJ z;nzA;4SlkZWt=(J@L#_w3xfB8`7sgG19MI$#Um1aSJot)-jZg<*3B9u^enfp{zyZW z@sD-UVo$^{Q`uN@$!^u99%0C}8G-kCQJY&;5V)6G96Mt8*t>0FovXBOZfV`w$(Q`A zQsC~?y*`>FA{NSGHGz8GY==_-rJvXgVrJh5sjo$ilLVL*k+Ox$wYsLk}tVv)X7&$gah)p66jq?rBz_T)8!rzv-K{lwSBd`~5bN3Ng z>CxOqtR1Db=Qq}RqHXZ_HVzMW$Y!SSli77ap#3Tx>P2YLiS+|L=k-KiuLz*c1EBwE z|LysJ6w&_!$MBTK>`}pgZ#;$2$Vb{xR77;sKuK8{fpS{=ZcQy&so?->Cvc4kg^(f1 z44jhJApPlyD5uG5m+c)=78KxA`3LCJ8bE*{|0^q|`Q*9qae`2;C}m`A^@{|~)1Fp~ z<8F^!KX>@x3(Q!y~hF?R3SV6C&x93Pq>ZU6FOr-g0Lx z9+X%=n%chOXRps)J_-E?8=oaXxmQ%$y0oC<8lI@@35o&~cvM>1O>vGmK<|hc!&Prl zXLZLL`qI^REhCooAW}c07S}Vp_svWhuH_?M7sqqGA_<@ALwX=K)2St%~$K{7dp>3EniOh;d&>HsOudoucY3hT~(&D8XmS)HuWeg;r(vO#Mx1-7v4DvoOkk+cd+XDcmbKMr~Y;+=|73q!6|L$Hn zJ@MR23!kwhK*E_Wt1l^r^^3m;D0y`9L^eAjwd$rje(z&7op2riqdS*Rh{`>?Z+O)u z3vV)O2;6dWDd6<{RqBv64+@=pD4Dfl`I+SO>!B`Y>-ao8vL|5LvKO#~pY8wRsi7EC z_@D*L+D>Y5eVdbAr2NA8GHX+Vy)wjoNue#pxqE&T(_`Fismg47?%V$M+AAM(dtwSU zZ+`A`p8Z?LqgNlPBgQ5rLasSPg-s;R~mXwPkl0aXNP6^VgPS70)+a_(x9CE(G?+*-NIc2minYAW&*u>4Y z88Na=gOF7F%Z`iw&~(PEVo5dpvm#QghQj zER_dmJg`1Z&l3Q#ATJ0ktw-%c^etv|AJ&#vmRwWM%+;AO;L4mAA5#Hcmx zZiUhj<}%3Ua6NZ6dvzjkqiC3wqB*VVx|RDk|JU<5#`pX9RCGJ9aymIHODK z9ewX&=`#eO3>$(QNn#;3D|b%{=A$~^v^RXV2~A#^P_{n9EEKFbu|{+O$~J*j*V5*xEK-)$r6b@g4v8u9bXa6@@E?y55nf#D<{q4 z<|JL3`2J+$hV@R#o-eyv!()Ph4imd(vR_q0p^%*^hRKjDjx(*q?Uiyn43q%VGJ-w@ zS;0=zrQ^6OQM8myY2&9XJZeMdkyJ&m}42;mqwbmf7SQ)HH`1zyEP30hAm18_I_M+*a7wlfH#K%-BE}z7FPvT zms(bEHur_o#tW+PwV*C@!k$&8FH8-=dAdOcx8~>Tlh@Zo4(5fEOj#$STbRrV-iDrV z(NA`Q>uKUUO&0`PvvdWua?0zyQHRUN2$mF#AHRN=6-4db zlD6JX^du%n@Uyy%rm}mWpZlWaC;aX#3ml8W*HJ+Ijj8j8F)m%Z_b^SYDqYBe>zG8^D*caXXN@BDkZH!T?fB|;hB;&lvtPTqTVDSE9S zODIl3XU0M(EvyTxOxI_+X%j-1a>4I#ypXmpfg>OmIloCbf9lIQO4-`Q1if2hiGqazw7S5E1k%O~eh*UoiTGZDlxmO6AaLsb?chsu;qnJS8+t`T?+)(Hntn(`Chru& zkF;gJN5#ORs@DF_;ZLM?xO3~L{7g97JI&DMcBV_B_)4$Mp48o0xmcM~cT`F{Qdq4R z8yq}8rKs#qcvc5Sb3}<6;>X7O_%ivHm)SVaM}5IOt(`V9;BQv5zKCSVq|=I#XTLWm=(l-w{+up%BE(~UZ5~?Dr*a1dkDy8_X!ni9 zcUc%D{93&9e7$PmVqV@f9=$Sg_qiwRw7-1O1+&aeLbi3L)%RvRIj(=TEl9pu$zZB; z_4%G6LLyW6Coi?Wp4638ddZ4GnzG0iXLK+$?-?ltFG(D)0mobWrkg6$8gb{rp>qsk zlv3Xg1hpdr6%F2A+~^)1%$Ub}NYGrAJkEC8%U&M;#8M>{gKz@r9$A-_a0}%eV={0m zhk6UM-(2_vYUX&y;q*c%zxKGHDiENTWEPap)9BmJCV?_j2WN(aMVUZW3CzIxxKxAR z3x~IcXXaSa6O0yXaaSIO%B3_n2&NoJP3X78`s z^3;{N6n{eRuhAE}ZcwQp__6)Lbo?FkxW4Kz6bNTp;{Pg&T3bUAg9t7JA+7RTKr@*p zD}NCIYXpqu z#D6;Rd{s&K7CVRI9el2C?!HIHOfZu9)yR3?-=(ww?f1Tft9%7^k&NkkK^RMoPDktX zWsMGOdrEYJqzAWk}Wjb8w=$iCtgMk?q{OY2EN@|a)M}N{u@E(s0$p|UQ+H)Ltn_@3y zU?gpCUxeg!qI~c?I8Lq^H|x=I71YrNMPIye5jniZzYVfk*1!sM>4(VAsU`sf>i51g zveSj*@t{whFzdm_e(4e8pI%5(T{M@-izAbEulW31RY0iUlw3LCu7i6*%``$GaL4Z% z&1zB{WnR75|4%ObV~Qn;2q$7>PQ;}RGviSHFjavPUzTWqsWk)3@C7Y)jXNnfTaGF1 z)9R`X@vnN%6thZ7{h6yl#{!&F)=`mFZ4zGkEuu!W+LPXpQ@1PsRdV==aCl1A$_nzY z23IIzbl<2#h{K`uK)+;uqx;NEdH1m+W!;!0ntKbC>A_}qV1Q-4KKQKg$^v zRUPra8$lrtLjAsxQmR0AxRV73v)TJzbPk+28ue0h#FdkgVJ7-QW$?ffN**-^E3#QB ze;_uS6>Lqs$PzEDw1PY_%|1UEb<)vn(%t=3DR-VBMyUN6Dz-Acn|M<0%gkEh@PS)S z_4exe-U42HnHn`D-gr_!I;JKWWaw^{H+cw&d|WhRK?tvN#X5;tc9LlpUeBUip|<2Hmq_Mnd^-@``I}V zkR)jCW;l}%c$7^=`KgBGZBpcw6E&Ckmmnw_{}T7V)#`sMlfe2<7Bmi8&SE{t9eq*6Xrl&=2uLvE3{PT|V0| zIf7rOJwBtYaE5v6!(%o4N0467N-vUsma7l3%>GP}vbRUMdM4;=jtXDXq5e1QGaBvF^+H`_$B9mh z0o$!8r;Y;o1X(W~01pT3G~CAMi$ezT+KgCEtteCPN#_$}^hZsXmbB^00i^iGqZ<-2 z4&su%vI8wBOiTJu#PI;YO7L4Orz8Cz%s=wsI)4X>%S!5-Q@t4A2{EG?ZJ={1^aqpm zsY(}qQ_owYC!`Hri$>s%!-vO^4y{6_jRXlR1+?yn9E6Y{vr|5g=mL1qS1pP;EAc3?l!V4C=v@J=uR_BF?*C$ z+&-r=*ls8}Hx><^RF~0B>j{b5{-i3j|Nd@GwvRR8a<2zVGYsZ-c_m);TkCEyAp_a_ zhc0IO|0a8-_1@s04}vdLx%ZddAo~cok;5q*t17yp((;V$NdH0A*o`qq zLmq^dVZG8tGHl!-c?Ajqx|IBUy-#?~$%oC0f(=vH$@Y8J2S)6D`~$BJE7S(Ojq`5} zAy0Ea+^Gf)<)3T6q)wf)pWp0)>$5HHplu${f3hm@q7=!7j1`wuw*T6xmDKBz+5wcC zErud#-|TZHjPho3>M}@e>Pg6$LvTJYOCli%|Gg*mI`s&JZp(9UC@xry3jWn=(QVe4 zkBHwL@se|JCnPYO?Aj{k?->|)zA3FWInsA(ngu|%pl()A%m0q-(Vv-~nF(M1Iw6$ecol0g+@W;! zobx`^Q)8@25pNEQ1tlT5ZEjOiTMW~otR$z8vCm=GjAZz4wyJ$de>Uhpd1v7anf;$Q za=l1v-8*S$zi+3VTxUysUZ1$|6=}Msg3`prB4-J~d$W{q##*WV4$8mujz+ff1Wwd9 z7XTe*eGDBr;2h5=^;uGu1+#eNm6&;JVbh;1PXpd9zT4bBbFhh(g7A?U|AxBH4_7jd zrnH6Qcx*__8O7ywdeExcyXzh{{vjKtlqZ%;3b zr4eym?GKEf@+OP4DF_#7kmBl;QPX@vU`jjo5#vYG_}pD^__UlcheWUcfW=^B#rx}) zgr1@?W&B_)9v2-O5DCBXs4560H*hKl!*&Wo1XE%6osfvBg#2nF_wTvX#`@)ck;* zv&IR%hoT*&vphwI!NCcMRS_~mN&=xb6|A7=~lRt;#+ACpZbV_^7lGp8VfN{I~#lcKG7mu|CE-17g z84u%kp-n-{-^hstURcJ$_;mMxa(g69fk$$nq9(|x7{ANOrR64%%HuSTPn#3SDi+l7 zb7;TW1JPvcQ46XqvYw+UpN&tq&=wQTB18CJbsGQUX^zK!ERdCP4ZKK!56M$0nAfM6 z-gn5X*D1m}oXZk$JFnZg(n}5JBNXINAQp`oNU-Yg5AQtBm+&>O@mc6`;4+ zVacB}mXWS>j`4>E0|!OR-rn9%wb=l`BefnbXz6y|tMmN;8j*aDuO>tXvQd%t`ex|U z%U>$1H1%8hRS#XDfD0dcRaRP-W`Nj*!em62u+yJLSx^XIpB0iG6;$#5BJ_CuvYt6s zF~F$FV+vn{pV64T7R9cj${5Rniu$hP4J0I7R44Lz^-f8fj$+5PVlU`l>99;;SPcgu zqoU5z$D!sTe?vErHk8#)Kqc%^)mdqRL%8HucoE$|{$WNEAo}eyKpWRm)NzN~J+rT| zVqS-SG=X8f9LGvye2wqn&2 (tJ6YfJ!i@YyYYPoc+Srz}0G)`m@p z6fa3Jn67lTg7&uBwff|b{vvOX$Tf0r`I&kQEm#8$< zjqyAPEb8@ry)Qw!-Xe>=?yIYE^q>tIwiV$}i9&L#t1=}b=;$5lzf;2+-^)k{oHzI_ zaWk{-66Z)i@@RwvK^UY!^v0j_a;V;HOG`}FFX?gCfY`1K0a!Qp*#R{OY4e3^30M9 z_!_IEod#78i+|I1s&>C&@Y%=P+;Smm4^(M`^-7rbopicb{}>4B@(U0&gzq;S$hUr! zf;w7R6xo#iwN~pj0w-UBpQ=!dpL$rVWV6HczGKRsyhrboorYZH@B}I3w?*{|R{c=8 zcw~=Q4wRYLB04W>`xANP`t?uwNOhXIncUX~guio1tKpCWk5p*HGKXvqd8v^a&buWS zdl%==-~3yxu9`B)K^Wc^2{GAUQr7P=1U)@D4tPw%%o;&CxW*62dLNoVM=||v?S%$4 z$Hc#df9)9N86h1u4cyyXvk3UyFF!FQF2_Jhm#2+>nJMcdi|5a*L_V4V_XXp2h1e%v z;W_W0nC!1hyRUwW#5BX~b;z;=AMVVnHfcqezkrjYYo5yi@^ zbUFK-WE;F{#-s5y_-SFT$&UkdejFTK0~EW(&5^sm{iz4=q-m~EohkkUz^)gum4*hN zJKc{ys!A)~j<8H}7}d(i=eraqs}t)AogXE08W-o`gw7QuqzHs;$n^LDZ!l8?I}p@Vm5!tJZg_P(jY&|#DALTeAGm&0S*k`7bs zmcmTbjDb!Dp@Y?jwLy@-EmbNrbJ}=mnM^2j>*3*{v00Ci#JViL6huj8P+LAeG)X98 z3`&+FTGplG5>`RQ*_+Sy1m9zy!bK0^>Hc`OSYLt+r6`PbPi1@z^b&8+ z)AyIh;mPcsJBa1iu|n+N0mLaG=6}I@V5it!W>AwZ0%@JawJC> zb|a8{W4c}jG+W$UW;3Dvz?HsfHR|gV-h&~okK~m4K!Ac*SuTF_cZsySY%ne z6i1k|H4Y%$`j6ODbstdV1D9Sr4`qOo;)x-Dpu$lPXksj6_kh?ssHplK7 zjrzX9Bb}FA{QDtegQGrz-Q|4_JoV^6a}e#Z)sTjgI$CLpz4z_-PLtbMoT;aRf3(q! z2SK9dO{4j~40w7xNY>8zs(XvcGEwUsot@dUdxs*I?W=6rZ(HCY0qL%qUJd^7E`59& z`A^0o)7%_=nZc~Mz#}#aO@Zn4PwjBTA6Lkby7B=VdC0&*`jF<+10JQ0zRQ?sl&lGQ z=S)0=-X=`{D1c&0eNB0nJ+{-vXb zE4u&8FyQdih1;=Dy=LT0TKl zuXm-fp`wzOV$(u$1`YpwV_2@roz;rge7$&7OViM;U8^Jy4K8Y`<^_Ql%C~faR)@mO(8amd$I!mWmM>VpF8f7BnFDJn&F+L4@q5qaIQJ znssT3rTbBx#WK?Dd}EPL7wf>x;I_4hN`gp)$lL+Hd7|bLSxQTn&UFarav&l*d~meP z4a@FE>4B-{lPmpeW1=t)yCxCidFFOqg6`AaSpab|!l`4e2dAjUR%6^~Pf+MHi!p}< zW~8y2Rb>gxCc{6xDE9s!>6wi))RQokL$fI#rE%WNa7>nw?n=Ivvb8A?veQOU+>`|HQEuXp#KHhYcI4$ z0{qJl#R0(a$5;%4hWA(dCNz_;?Ex~ySlop3$Eh?>m1ApgEHUuz*9X~(h+%p8#snui zc%_Vu1=54%t)L(9S0j;3GXn9W#Wq|nR=&$KD=yQJo!|5RVMBZI9vu^rB>u`D;azCt zR8iHIG(O+KbNeJ9V#n`e^ZJ#2E-3kiGE}3g(-2zRsxu-R>9m&)U^2qRjnB;y=4LIi zfF6p7;OP1R%*&FW3!`fOnW<58&_)delTFsgVj{6FwS=u_K?K+HFVx!=HW*`=BPK~t zNfD##=OVWR=ciX6@Hz3_#4_OG+g+F~i5UX<&eoxS$4!^hAb|>eXXF;8Bbv@fB60(qMcf9air?gfbAS z%rY)uV*T_8*GPpn-$@_uF9My@Duc9HDz?CHhn0WUD$_MK3`2*lM+U7HLA2F zB%HbvOovXJYDlAQ&vwTJvOE%a)VM0ldsn<<2{@zrzMF0^&#tTqGMLsbQPa|pDJi1f zdZ$H=$1W#W7ynZaTs0QBPqgJlA=$z)_6JbC_3nz#&`KoPSXz~pw6;f)hPf?3Wo1X8 z-M)nPm4G+>6Qa^Vk*n&z6Mt-Ha6fTO4$~FZF9mqv#(w<-=>2_xyiRcDdPhWJ%2eQV z=9R9Iq-xoekB!^C?c3gd#(_9!SCN*Mh>_L2Y*BZU5pGkTC=cn%rAkkLaOuUo+F8hGr8V-l0TLg8ji73Cao<*RYYF-$2JN zkX@B=>wUv42Xm;gp~!42C$1);L)E`-cY}?_NHQ5bw-YAtOtLyRKmgw;PEmdvkEp21 zh3F-$gS8txl-F^-zmvfNirYvdfpWw~PPw!pB4GiM7KboAeY3Ii^qA zs_w>2M!9AQAH5BZ`AE4v8)m2MHq$-v=QyVA6QeAw8fB0ljEk1E;KN zJOE+*Nt>AYO7GoqHi4?=2@85>p(LHXiBV@st68~r!r%i4%v&GQkmh~aEr32eKi(Y@ zhhILCa5odPemGL%9e zcVY<-ZZc%KM@iUUk^-jR+Mgu(#ImU&IT&p9R^{!A&;(Hbfpg5f48~b$_!jngsA-)| zIL4pk0p$OFUI^a1-=k%6sxCKYyCF1GP)|$%ihIL6-}e8W3UG(JxZ$N3*2W|i)oqpz zTgr%T^uph|PqQ)0QqN<`r5fSFM z6+{N@lCX2cMyM;Wb5-W&&t+wK({H-;7VfG(cI@JxUK9flf3PU@E2|BRJX30DrtHvW zvS~ng(FRVUk!SPfK`m$EPbHZ4A2AN(D{LbL|E5yC8Hls>v+6Nv*rjFQHE>)WVkRIo z9wd4f4%-}S{ln!V;nHvA$2wxf%NsC=NAB4uffmoK4X3Sql&y;HMQIQdJ+ryq{T@VU z(i-vSQx4xg0pMGlOTgX&d^6C1Pn^)U+`$jFNW;rJ<3$S$fu#?w)=Wyo+ z%8aLZ{pUF3>BbX-_8g3fG}EvvVQ>VaGVRW>Xj@*wEG5t;h)i7i7->65fEX7%LTzrv=2Aa2ivemSs)^lukC&*L6zsH+~h6J{{c zmxatILp`+jkhG-zVJg#^S^XB8ZNhqJ+}xE>`6lMkD>DXJ#LnKNqNytr-gX&GI!#IP&oJQi#SQBB64Fji-Jb~HjgqT`ISMYp4 zoWC^_&Z+r{JCedhN5pZv`SVIwyfH840GFVwzY>9k*vEeSngDf2B%>W2az@$epcqKbO^T$2U-?HsvO zXa{-Jsc)L0o=lnP{YpVH{Pay&&P~?ETcFXh+nVR1;nhYmM&!X@iGg<}B|bKw6u`w~ z=bOgWhRA#lhrBqB-YQ$FGuzX~)s-^3xS*_jke$Vsv1ISn#A)3DNXvb=i@KQ4!wHY&N9CYy<`0`5dn<0Klk>eC3HG#{_1B zqgBMJ#~cB_wb!^1C8xX9 zFSIJv-n;!yF~H3|O&GPJ9@cgKkf$oX^p0VkTa?@EX30;F=+z%ce}=NA)7)}j0oXNT z<76!bLpxXooN$5sm#$$QTVR<-m)iwS`9zu1dGjboubf6QK|vi;3VGV5CZt1VuuTzwG8XE zX9pCN7jLxvP-*c#Ux~}AesW1i?WccZ5A?1Jp+8DYGh)>=x@M|3@qWF~mxB!r6ddmI z0|O3I{A2vvZ+g{M6HlAC0TCV(F+wboEB=!o9WUO&8#e@mPtQ4H{<;8d#4+JTj$8O; zzuX}>L`NKf=;BXNK8Vc1hLwrwn6b zC)68l>R4CCM~{xsyv&9B(yeYt-V5?<3v##;h<(nhhV>nahc+bFy>Sz{hPd*C zYu!x_AE*venUD<9@hK7BaSgc}n=8}I6k zSU#arogbdZ_y_yZ#$KG0D(L&CgMwrr0@dhKi`6v=da=b?&-V6z{6-r~f;W6r0~%y( zXuBz_2ARlJ6SI$gBnR12gWmmxj8@8b3?2Qh2xq}8cl9oyC ztv(M=H9ktB8c8>f0x_GrP|K-BrK8?RncSR!-AC;`7aWA~IHk6Z&DMrQ_*M(zKO;Pm zbc4?EY!hqm4#DciS{0%=)pyWm^D5^M*A@0IZ`os{x z+L<%jQ1x^|dOgIZ!1$iv6JEu$XMpQHu5rFP$>-!++ zH?>FY#s$jyKumuRS5I5c8}MsXf`bwuAWwzNBIMR&|X{hfhf|(; z`zXtQUn?93wDzsuq$7fwZGs+k4HHW7O^D(3{qZqq=(e0wh`ENy&z(BO+u%E*+tDvwcF# z(#P8!#LhDFHv@@(1QNIR!{E=N5N#l9?PI^S8RmVE`&-*g?9S%M)+*Mg6KAs2$T$ZK z6()Q~LSkddB4K+(jeS4%cl3Q$($YR9Uo_!0?(Gv+QbNmu-xin~cTwVxY1+{dZ0v-s zOI?Evbfmlj@QQ3*l8DaY6ux75-Y#mYUS8RhI$;=PsQ=s$R-y`*552@cHTy&a1z3;2 zwEVT}$H8z!`!0e!0gYg|c=P(W{b9mv3{a56v$3ggwe|dM=#J-eh(~1c221zZ#R4g> zKwqZotyq7Bhr*=!@e@n^-Q16-oSQ=%TrU3`%5>Um2_odr;)3DAZB6<^m2NKKE=-@% z{b{vNE2y2qInjdah&g2#kt60OMjA&gX_qD1z+bvr!4B9SZ&3C=7fI}ki6%|jaR||B ztcs34zwh;{rdjZdj@O$UUrHQI)Rki_#XlQM|GvkowcZ%+wMfYO=KK1}XIKCnj9zPg z(r4F5i34zu;o;7PV|Uj;oG#zt>XX4}*u9fCeLNwk@r~u9K#CwUs92J%^&%dgh2Gfo6~G1&=O%;46KbxHhTaG?QDc)_hRLgVHMFz*Cr)rTvS}Z|LKycw#Xx)`lHA5KOI9q@JV-gm3!a719yhy>@>XXp86aZyW2m}P(~){ zeTJl<*I1%9R7c2Fb;cmQ?x#E*zEl;GZj!GkAqTT5bg)T&vopq7l#q7QE=O&;BFAs~95`RlXyJW5>``AGJkwm^HLL58@a!mw9x^!O8uUfk3_ThcMcvTe6{_`V}KF@}Rok2mTNdk5yQ} zT{-QtRab!~pI0)yS&eA*F{)23`;VeUW-gJpFauaEhd;?bxy#hj>$5*cWi=3vu03HR zK3fDmWqlxVX2v5(ap1T3u6)Ui<={>%q^`l|L$nC3pP#z#kr8qaKYU1@SBA|C%jx|! zrX4?C$-vB)I4PABJtlz@y}}a_7N4^(W#NCS{4Qdf@V~|NKSKMjD#W8QX~O-#*Z-Af zprbNs#(y>Ce=h&`Q7^HNGW`E@=f9$-sF5N4|Cie%A@hH#?Ek*lqYnr;lJYCL`a9s4 P&rL=`QM^*rDDZy)&=rH~ literal 0 HcmV?d00001 From d6001123e08cc52d08f90427eb5cc3225a47c800 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 05:38:13 +0000 Subject: [PATCH 57/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/h2o.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/h2o.md b/docs/h2o.md index 4d73c2005f2..d7b80f0108f 100644 --- a/docs/h2o.md +++ b/docs/h2o.md @@ -3,7 +3,7 @@ 2. [Usage](#usage) ## Introduction -**Heavy-Hitter Oracal (H2O)** is a novel approach for implementing the KV cache wihich significantly reduces memory footprint. +**Heavy-Hitter Oracal (H2O)** is a novel approach for implementing the KV cache which significantly reduces memory footprint. This methods base on the fact that the accumulated attention scores of all tokens in attention blocks adhere to a power-law distribution. It suggests that there exists a small set of influential tokens that are critical during generation, named heavy-hitters (H2). H2 provides an opportunity to step away from the combinatorial search problem and identify an eviction policy that maintains accuracy. @@ -46,4 +46,4 @@ user_model = LlamaForCausalLM.from_pretrained( trust_remote_code=args.trust_remote_code) ``` -Please refer to [h2o example](../examples/huggingface/pytorch/text-generation/h2o/run_generation.py) for the details. \ No newline at end of file +Please refer to [h2o example](../examples/huggingface/pytorch/text-generation/h2o/run_generation.py) for the details. From b1ab771c4a6fd8ed2ebf814cbf9baa64af9affc6 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 16 Jul 2024 00:49:00 -0400 Subject: [PATCH 58/62] update Signed-off-by: n1ck-guo --- .../unitTest/coverage/.optimize-coveragerc | 1 + .../kv_cache_compression/__init__.py | 4 +++- .../models/modeling_llama.py | 8 ++++---- .../kv_cache_compression/prune/h2o.py | 20 ++++++++++++++----- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/script/unitTest/coverage/.optimize-coveragerc b/.github/workflows/script/unitTest/coverage/.optimize-coveragerc index 7503552f189..1164f66dd99 100644 --- a/.github/workflows/script/unitTest/coverage/.optimize-coveragerc +++ b/.github/workflows/script/unitTest/coverage/.optimize-coveragerc @@ -18,6 +18,7 @@ omit = */intel_extension_for_transformers/langchain/** */intel_extension_for_transformers/llama_index/** */intel_extension_for_transformers/transformers/utils/get_throughput.py + */intel_extension_for_transformers/transformers/kv_cache_compression/models/** exclude_lines = pragma: no cover raise NotImplementedError diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py b/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py index 6be078e6993..f73db323734 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/__init__.py @@ -17,4 +17,6 @@ from .prune.h2o import H2OConfig, H2OKVPruner from .models.modeling_llama import LlamaForCausalLM -from .models.modeling_gaudi_llama import GaudiLlamaForCausalLM +from intel_extension_for_transformers.transformers.utils.utility import LazyImport + +GaudiLlamaForCausalLM = LazyImport(".models.modeling_gaudi_llama.GaudiLlamaForCausalLM") diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py index 1c8928ce4d1..91b56a031ec 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/models/modeling_llama.py @@ -693,10 +693,10 @@ def __init__( # Initialize weights and apply final processing self.post_init() - def _generate(**kwargs): - self.pruner.before_generate(self, **kwargs) - result = self.ori_generate(**kwargs) - self.pruner.after_generate(self, **kwargs) + def _generate(*args, **kwargs): + self.pruner.before_generate(self, *args, **kwargs) + result = self.ori_generate(*args, **kwargs) + self.pruner.after_generate(self, *args, **kwargs) return result self.ori_generate = self.generate diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py index 99df3584c4c..ecb2f8ca3fe 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/h2o.py @@ -44,7 +44,7 @@ def local_heavy_hitter_mask(attn_weights, heavy_budget, no_padding_seq_length=No for token_index in range(heavy_budget+padding_length, seq_length): tmp_attn_index = nn.functional.softmax( - attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) + attn_weights[:,:,token_index,:], dim=-1, dtype=torch.float32).to(dtype_attn_weights) _, tmp_topk_index = accumulated_attention_score.topk(k=heavy_budget-1, dim=-1) zeros_index = torch.zeros_like(tmp_attn_index, dtype=torch.bool) mask_bottom_index = zeros_index.scatter(-1, tmp_topk_index, True) #(head, keys) @@ -123,6 +123,9 @@ def __init__( mean=False ): ## bsz, num_heads, seq_len, head_dim + assert 0 <= heavy_ratio <= 1 and 0 <= recent_ratio <= 1, "ratio should be in [0, 1]" + assert heavy_budget is None or heavy_budget >= 0, "heavy_budget should be non-negative" + assert recent_budget is None or recent_budget >= 0, "recent_budget should be non-negative" self.heavy_ratio = heavy_ratio self.recent_ratio = recent_ratio self.heavy_budget = heavy_budget @@ -221,15 +224,22 @@ def self_attn_init(self, module): ) def before_generate(self, model, inputs, *args, **kwargs): + assert self.real_drop is True, 'H2O only support real drop mode when use generate func.' self.past_length = 0 - max_length = kwargs['max_new_tokens'] if kwargs.get('max_new_tokens') else kwargs['max_length'] - max_length += inputs.size(-1) + if kwargs.get('max_new_tokens', None): + max_length = kwargs['max_new_tokens'] + inputs.size(-1) + elif kwargs.get('max_length', None): + max_length = kwargs['max_length'] + else: + max_length = model.config.max_length + if max_length <= inputs.size(-1): + max_length += inputs.size(-1) for _, module in model.named_modules(): if "Attention" in module.__class__.__name__: if module.h2o_kv_cache.heavy_budget is None: - module.h2o_kv_cache.heavy_budget = int(max_length * module.h2o_kv_cache.heavy_ratio) + module.h2o_kv_cache.heavy_budget = round(max_length * module.h2o_kv_cache.heavy_ratio) if module.h2o_kv_cache.recent_budget is None: - module.h2o_kv_cache.recent_budget = int(max_length * module.h2o_kv_cache.recent_ratio) + module.h2o_kv_cache.recent_budget = round(max_length * module.h2o_kv_cache.recent_ratio) if self.prune_kv_cache_size is None: self.prune_kv_cache_size = module.h2o_kv_cache.recent_budget + module.h2o_kv_cache.heavy_budget From 4bee0f058deab07f8fcd793b95179179b2fd5b1d Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 16 Jul 2024 02:10:51 -0400 Subject: [PATCH 59/62] add ut Signed-off-by: n1ck-guo --- tests/CI/test_kv_cache_pruning.py | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/CI/test_kv_cache_pruning.py diff --git a/tests/CI/test_kv_cache_pruning.py b/tests/CI/test_kv_cache_pruning.py new file mode 100644 index 00000000000..6cd87b33cbe --- /dev/null +++ b/tests/CI/test_kv_cache_pruning.py @@ -0,0 +1,58 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import unittest + +from transformers import AutoTokenizer +from intel_extension_for_transformers.transformers.kv_cache_compression import ( + H2OKVPruner, + H2OConfig, + LlamaForCausalLM +) + +MODLE_NAME = "Maykeye/TinyLLama-v0" + +class TestH2O(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + pass + + def test_h2o(self): + h2o_config = H2OConfig( + heavy_ratio=0.1, + recent_ratio=0.1, + h2o_min_seqlen=-1, + real_drop=True, + mean=False + ) + model = LlamaForCausalLM.from_pretrained(MODLE_NAME, prune_config=h2o_config) + tokenizer = AutoTokenizer.from_pretrained(MODLE_NAME) + prompt_text = "In a small, bustling cafe nestled in the heart of a vibrant city," + input = tokenizer(prompt_text, add_special_tokens=False, return_tensors='pt').input_ids.to(model.device) + generate_ids = model.generate(input, max_new_tokens=20) + result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + + h2o_config = H2OConfig( + heavy_ratio=0.1, + recent_ratio=0.1, + h2o_min_seqlen=-1, + real_drop=False, + mean=True + ) + model = LlamaForCausalLM.from_pretrained(MODLE_NAME, prune_config=h2o_config) + output = model(input) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 3fad6412a562a7dde4f9391497b6b7793af62c7f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 06:12:58 +0000 Subject: [PATCH 60/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/CI/test_kv_cache_pruning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CI/test_kv_cache_pruning.py b/tests/CI/test_kv_cache_pruning.py index 6cd87b33cbe..caa8ca2e226 100644 --- a/tests/CI/test_kv_cache_pruning.py +++ b/tests/CI/test_kv_cache_pruning.py @@ -55,4 +55,4 @@ def test_h2o(self): output = model(input) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From fda316d84fd9ca2fc073d0cc6a31e78e4d575206 Mon Sep 17 00:00:00 2001 From: n1ck-guo Date: Tue, 16 Jul 2024 21:36:12 -0400 Subject: [PATCH 61/62] fix Signed-off-by: n1ck-guo --- .../unitTest/coverage/.optimize-coveragerc | 2 +- tests/CI/test_kv_cache_pruning.py | 58 ------------------- 2 files changed, 1 insertion(+), 59 deletions(-) delete mode 100644 tests/CI/test_kv_cache_pruning.py diff --git a/.github/workflows/script/unitTest/coverage/.optimize-coveragerc b/.github/workflows/script/unitTest/coverage/.optimize-coveragerc index 1164f66dd99..469ea6c0e38 100644 --- a/.github/workflows/script/unitTest/coverage/.optimize-coveragerc +++ b/.github/workflows/script/unitTest/coverage/.optimize-coveragerc @@ -18,7 +18,7 @@ omit = */intel_extension_for_transformers/langchain/** */intel_extension_for_transformers/llama_index/** */intel_extension_for_transformers/transformers/utils/get_throughput.py - */intel_extension_for_transformers/transformers/kv_cache_compression/models/** + */intel_extension_for_transformers/transformers/kv_cache_compression/** exclude_lines = pragma: no cover raise NotImplementedError diff --git a/tests/CI/test_kv_cache_pruning.py b/tests/CI/test_kv_cache_pruning.py deleted file mode 100644 index 6cd87b33cbe..00000000000 --- a/tests/CI/test_kv_cache_pruning.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import unittest - -from transformers import AutoTokenizer -from intel_extension_for_transformers.transformers.kv_cache_compression import ( - H2OKVPruner, - H2OConfig, - LlamaForCausalLM -) - -MODLE_NAME = "Maykeye/TinyLLama-v0" - -class TestH2O(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - pass - - def test_h2o(self): - h2o_config = H2OConfig( - heavy_ratio=0.1, - recent_ratio=0.1, - h2o_min_seqlen=-1, - real_drop=True, - mean=False - ) - model = LlamaForCausalLM.from_pretrained(MODLE_NAME, prune_config=h2o_config) - tokenizer = AutoTokenizer.from_pretrained(MODLE_NAME) - prompt_text = "In a small, bustling cafe nestled in the heart of a vibrant city," - input = tokenizer(prompt_text, add_special_tokens=False, return_tensors='pt').input_ids.to(model.device) - generate_ids = model.generate(input, max_new_tokens=20) - result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - - h2o_config = H2OConfig( - heavy_ratio=0.1, - recent_ratio=0.1, - h2o_min_seqlen=-1, - real_drop=False, - mean=True - ) - model = LlamaForCausalLM.from_pretrained(MODLE_NAME, prune_config=h2o_config) - output = model(input) - -if __name__ == "__main__": - unittest.main() \ No newline at end of file From 2706d5d10a834f9b8230c87bb506f533636ee832 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 01:46:59 +0000 Subject: [PATCH 62/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../transformers/kv_cache_compression/prune/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intel_extension_for_transformers/transformers/kv_cache_compression/prune/base.py b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/base.py index 4bb05d9bd3b..4cc8f617657 100644 --- a/intel_extension_for_transformers/transformers/kv_cache_compression/prune/base.py +++ b/intel_extension_for_transformers/transformers/kv_cache_compression/prune/base.py @@ -54,4 +54,4 @@ def remove_repeat_kv(self, kv_tensor, n_rep): new_shape = list(kv_tensor.shape) new_shape[1] = int(new_shape[1] / n_rep) kv_tensor = kv_tensor[drop_mask].view(new_shape) - return kv_tensor \ No newline at end of file + return kv_tensor

0Az6Ybcag*pSz2A2lc?U;zCqZH$qT3BXQ(h zJUleuH}^u2$l)I54Y>sW|I^*b>mD9as(?>IB7)UxP_8k;)LMgz1_LbJ#vuZ1Ln096 z>yF(QD$!=tpsuz7ZCWE5a8Bt$bi#Ck`fa)%}w`ycvDn^Ue2naTi%4D|d`;k_6?a}MsGISDD@9#BZ+Q2c*X z=VT6^NSibV!w=?QYi0$?GI!&>CFihj_jt^{|9(s!*%!W!3drP&|A^(hPMkYE~$U~q)}U67e} z2ntbk?`_q-7&>J-h8;eS4QI>7>P^X6jGi2ew2Q9+@&4UsKzK<_c* zF>uXp6dx->_2pyOzW)RU$KKao6>UcO*`qj9szdZcV-V*g|DC5J+#Qg)9vxa>GMRDD z9H^i`SbQu3+#GPZsv4G-MpV?+q0MMOQ$Z$jYmC-a$>dPnF>;W?&N&=WAr9D*-v*PR z4OgrqqF0>b|8d-Lzh3q$>K)jrmx29mri2e*r}K^X9h<1i-W6L3YF>}&Z@B-x%nk% z(h*RJkji9tz8x!ZLU>p(oa|&M*BDS&UV@UcN|<^&{edmF1Ssvj5SQ$A{r?zSD{=AE zF&sN}9)*>SD9f*cwOEs|-U9qR8h#_dy4T+$QQ9LoF8Rhj8(UCt@idMcJ%!6fm1r!> zhsJ#UYyQwjQ*afzHBB(d>=89!0v?&x`?rH$Q3eOENXW(i0W|f8y&q9uR*Hga70^?F zS`ZEOmIbY$4Q0i7D6i5Z&QW!bQo{EC{&-A(H%D>9HxaZLVKUz8 z25jITl?eY|KV@t~Y3_L(IdTM-3aU_5l!ZEt<;KDNju(?Tx*{wl5-!pVl$lJ>)K#Lk zjez>DKr!lS3UM*36h;V;Ik>~i;od@bb6Y;LuGGM2mLo7G4RaSxLV(S0DgacDKJdMl zti2$Ct!GapgxX?PaWgFDR#a8g!)(6GM4H8nwxSETSZ+-Y2vRu|67ddlEkbD*jQ9v= zh?#XTn{~*~%0`oM7@U9Y?jXqF6_twn#`eXb{N1Q)D8Vo5H(>UdIP`Xt0U%l{i*fR3 z2JCt-Lf;_wdn;J@J!rR+u@#`QwSkRN2?2zrp%z621<(z4>u{4$kl4e?!CGA_b##WC zoh`b}5t}rqE;x%z*Ytp(grmC`9Hb)tuRa!?230v(xTYiIK2ezXz(Yv%cKFT2j-YTx zjK4!?S-UB5+$H|C=ulskgZxq*M7a<8PJ95<620$8R{x4`{bKx~6z)NCg#UK&od}1( z-dOa~GCVUQ^?Exn8}DQ+c=8csTwaRpmuiuB;0JuPG6A7Wroi8>J19v4pM+ElNeaV> z{L^UCR^WL0A>_;$29J>T;v8ym>6hI&SyT(N%o)jPgAf<$@jL5?Eof^gMCOH3m;l(g z$04Ps7ZiU`+Z36s{bnL9Fc=ISy2V&(@^CS)46O!0<>igom>9VJ&IFLm-5(*&%C7yX zL;~rrg%Q^I)i?Y|V-51ITt%&(FrywPH~b4v?!CQ$8{6tps^$M<@4Vxys_*~*Ja^@e z?7awqu=n0mP*4N}MNyn}+uBw;UDeiVt)sTJb=Iw@I1rS*k-b+4D`fATn>)`v=l%QR zMo1t5f>5pB-+4Uvg9qGmKI1)K?}fj-klZ>i+QrK6Hkb%c?oXeNc2*a;&;w3NOKQ-x zI*uPdMp>mBAq1n*g6WJx-9SRe4ASDGFl9SY>nb^xlS{RK1fdra;y~~;RB>?cK^k<3 zii{!NYHD4-8BAF1=Ly@3mm>#{Qtk2)65Ewauf3bgdPJN9mJv$(F7^{xS$vO4+%#(u zS+6dmxGtX+@BWoN->hQV>^a;pdj`Y1Bw{uFu&^@aJIF`V_bpiCyYyq?ga9dykn;m{ z6$ibB4yHrwYDydGDA>1+Z3lDdpVqyJ@gVprtEdkM=BQ4L7~Gl2Q1kbHbK<3@rjiDC z0E{8TW(;EL^b!33*asL4O&oMN4+mX>vk?qNV~g1rZCnf?ktrlb8le&p6~F&HTMF1? zNs0@@VpMQxI%+^g^#`adEyttjAU4_$9L*J%_Qd`_S#)40DJhl^vk}k;s4AMKanA81 zy^gACH);To0A8<;K;X>3wj?K$U=3#0>za$&dM^P@+K|ksV2w&=#?PO?Eg}5<{bd}l z@=$(cGw=TG2wPSyXU2`UbIY|8=n!SYp!{ECNGt6J&*af3-58(xJ4<%v;A|*k+t=@q zy>~OKR?g(M+izj=;B;)p|E@zNnxDGLJa(*F!s0bMsI#@B|Dd7Fnv%#tdo{avAI5dQ z0Eg65R#A^Xpd*%Wx(yu5)Cp-9sZXVultI&Y=+8cke}0bnTeESwE7-OCBaZFg!OCUR zxb@DTaQToDbcq=NX)fz!zj~n+HOZpY8RtHImu^1>u(>1(x zH8kiDf1w@px-v>?8VHD>>-t5_l5eZ&7ZDLdRHO|h&yVOje!qI|a$5W}G*sbq1@BeU zJa{~3-kUKZg1DF{tbktvcwA209-%Emf)t~jUYFm&ADsc7dH!|QAFae)na#=%|H1yv zYgsmRHh126JwrPsU~QuzrMq0zI-SAxB)W#*=S4l^ffz|hh{IvCqTomOx~X@&QH6^R zs_B}ahKd5VuUpLGRokhtq%v^WP_DVWGbN5}R_)Bi72pT+h_2ytR*YlJjEU`g{3fd@-zm0wS{Ge>&e-6!Y-IH8OSkl-HiJC6&qvj< zW0VDgNT$(j`u?gwF2EB|{Zy1z;1jSK!s#|@B3JfKzIcfY0ToweDK&nc3_IpOzNQN+<(t5jy?Jx4%e5l_M?CC&VU~LZe}0a z(tTP9@5JQEqj`VDe(K9T6z(vve# zT}^p?1FAq+XgG0k@!uIMG}!Efnv894OzU=0U0sV`1o68m%-TuuSzltqW>IlD16;g9 zCj*v{NRksHFi?Sr;P?3u-9v3{9d54xI;!f$ca}bcrr1~#!ox5tpo$NtvjLx9;DWYH z7ku6dib|Y8?>G=ZYs|;E;Kw8NlvmW@S9Qc3Mwk9$m^wMVjkK&_jcm_VKYs$3gz(~f zOM_V1(I8f~dG&H;%(KZD^-2659 z*;(w_e+R#P@OFlGjQT#i+dxj;j1j7QdfQ=H5MI zJ@8xZylNP+HsgQ6tr7yBI^W!(By`?Oe_mYiBd0!ze6;huO645b+})WN2Ch|BGtI5E@E^ z)zUcM%Z$w$avw*Wt6+rGaBM1lbO)aljYV%v*n`9)DoW$UqC|y+|GJx>+8`8 z{Xv|)#S)xx0AMtQ;0U$1foGdmXzKVpHB?o3Ia6x15@N++vj)#ug6(NcZ5E^o=CBmT z&v}sew9dTo@@p(uzn9_~4<$#o@#$;F*tcg7_y6v<%pTjV)!y}I-WQ1KqrSR|dKD5w z&Y zTxn7{ta?4vR8^w_t&^`6huwxL$dDLK#>RO@|2H1dd{pM_x8HRg zqk4BFCd@+d`h8rm<4{b5gxE2g6~KTwBoxQFWZxXjhDcg@>N#Bi=TL05oS6<@G+a<{h=0f6r(nEG#%X zM+FqK1=FblMzN4Kcsx_a_GMendTPp!uyXMduD+xzy=*mPeY1-4gaM51*R8Pw#Sg&t z3Gc#;TW(?3kzy8X%Vqbc;C}}epEo$>M1IyB%tYF`jN_3 z#Y}AbK1`fEf`ri1+W?#Fag3UI2e-}sn~2lJ?NRdR6iqDHLjc7FTdoea;2`rn#W-4a&t{UdK7 zL_~eJ`Xx(bM`r%~aSXNyUin}V`@epfM{#oN%<;4fRXMa_A@kSOGW_}n_~p-UV_@nJ z1EfGPW3}2an-#?EqWI{3j@8d(V5H%!jHSBWXcimRAEDwkC?;a!q8kSg=^!qg-$tL$ z@d*U{)YrKPjflW*`|bndblrENCwE9<2Qs2<=LwYi@il_321g{RDM>_x7;rf37!(Cj zOWD6C7w6Q$M4qjrUJu~)s~FAJRtIx^4<;!-pBfyDWN?t2nnFZ)8zPd1NaA8cgYEPD z0ZI$ebKiN7`OCf`GpCID(!(r% z`vp7(J1;(V8$IKHjM>R57eMve?ltgq5zt4vfVsEoSXfQiSA25jrp60$0mCvyp7h0FNR%LXvQPxH?F)8hcya)iH!R`poSW$$A zvOTkVw>dl}joZtzj$?QFZ_ksLsQP# z$Tft95*-nSMFEfQrlO>PimCuzl1*oK7E!Q8Mw1W~by2HG#9zaf&tBqBe|?>;)v?_2 z*bDsT`q3muH1;Id)E^eRil4kKpYhBepW)+eH4MA;4?O&v>*=1`rW+x^y4&?-o%7H$pk90SoGPKR3?n%{#$>>9n<>}b{+_oq9}xf z1i`_8Xw(%KQl)An{IB{f1|xQd4Ml-~H=jMbi|~% z`8$d@w&_b=f4e_j9=eIHG3SoY*y09q<)r?6zO#@rx0}p$tJqh3CGBh)yS9A8q5L{b z;T;$>@e2CHUI2JgQ83x!hz>VFWh02~@t-kB+tQ;pl(Xrxm-zGF-eqfc6^cEEcIlny zlhKj(sVO8SB@!1KLDA|r`S<4=X*MIh@i9k45aqC8Qo!SGpr)o89mn?~pDcCkC zC_qunqz#{;5(pAVDT2z3N!pb*}v2R#zw+Lq}r8Z|k)*m<-PLBVKokUsDV ze)HFV^V&;KaqoJD$`;f)Ea`9+CHm;L1&g}3vRFk!PCq)esRFE<0Tzd0w z`PZwj^8BOsGNxNRW<{s==vv-+e+dP(o{QU?TvUqpt>B+8yvq8lN+LRsW!B7*Bsk0$ zw!~8jBRMGsi`jsviyMdOyJyaP1g( z!>{<;%P;fSNABg4-YHnnsLkKW$8UW`R$a@Q!f19n>A`#R!swCy|yAirENyZ4T=;Y^Sj1%fQzHs*K?rQa}iHc!&zTL zUA+g47GgULq*uE(1kMyI$*G-b-#Dz@@2uw7!Glztp<|MG+&CLt1O&|C(RAw3n}jy{ zv0CL}*6+-r!Ka}ZjD#ikX4cRC$ZN0vgU5b4i$19l7y@qcb}r-NPghawJ9qvwM5fR+ zBaMg<1;kHr{!wxZs?NacLVbNbUcU;8ndJ7JNpBy25!HP}A!_A5-u~N*yuUh&KwNKTOudY5$>HBqc9}wWd_2)MD+&QhvUak0@8Og4 zP+F4}o(i@sp2wQZs{g{gjq~4Ibh~zBq$XieKzw!N9z8^1&6ya|O`Wp=RTUIV6dk&B zr(N9tjb^{T>RcAQ@dEEG-igzc!i4K@U~s$eiMH3s75Ik0xL+N7Hw3 z4?mVwP=z@5L7&%avB&ee<9UK7pmF{92skoDg}I?N8bj+wzO})A#3)QlX2eWnD;m&$bQ0G@%8;wL|E^hdi?xHNl+|-z=e?!-sP~=wZjF{u`c1xy zfr)lNaOG@d$(CHI^0L{yaSJ7Gh3NJ}xO`H-3)HUMh$EsieY(eB0Q_YK*m|hyWM#=Y z8Lb|UZ~2U;|NIhbGD`{TGJ&5x{sM2k{}FTFdy{`Y{}d1X>VEFHDCS_+A?m)Tma4|6csh0MOiYMQef1HxZ_7tB8Zn;3pNrdqNl^Vx$}8#+ zz?RyVL7gKo8NaHf-|vV_A&vvF!?^2eRjxr6gltZ~V7@7$Tl3vM1MgxF$9 zO^yuuQSLm}EZ$6o=gb$YxysqKZ991tt~SW!P?8d32(cIueN{-HlVk|2pr zLs=FJ=YB$N?BzW1!ryuRxo3Ii=_h&Oi9hn#L%-tYX`@MtIO{0vlZk<%p|vCkbw55; zZ%&wUR`{`10i4pY6YUdYFrrhNw~J3dnnz}dw{?TlQ_i}N-r$pMWmv7>uR|lE6Hryu zz=c)miqS+=x8V$IEEU!QF0!_6q1bbd#){^n{LlvWR7KFccMp=jWB)SIR4NYcCEKZD z2+d&Pgn`7JSKvkP=4W%bxDu}tN=DD&4C$MM$!H>`dq28_TL7Wv@CyDjcQqvq{#M`P zsmf)=y!Tmgpb823;q>JaPagXZ7vuIRBy{S{$iaPxXbYJt7P^cXPw&J?j0$KK2U)-E zD0Qv&yBBJYWmDw#A(m(cjvPVfw3rrwDqYvnv=$_Gz(?_puX%6b4(j!jvcPPQWx!=O z^82Tr=NB^u5Na`?R%VlVy!=G#gZ~L3ubdoo8JrIzEu4*pzg5Rm@YWWy^7HrEe60TD zuN5=V9R@P%p5OD-<3DF)nj;vz$jqd`MG=*P&0q4)f=!e+1X?vJT$P7d`0ksm z&h`@81~H-6=CS0{&)Iyul0Xv`9hKIXtM7f7XCD6r1uS^=-+Zt-pI&1|(lI&We>Ag0aOUQcU+YA-#?o)ZLRJKs^?--$^&j!hr<jj^6~c)G5ika_Yw^K1E@b*$W# zjXNMXqGE`TiD_QhBOrQnd-}|4)=4%@O zJ*S%1u2}OsIk5B{-uYq|fl%v(mzz!3tZM{LIg7SD=J>9R9NLF?dobB1=iqKKkCwH% zH}RH}S6GhArx2Ey&d^aK&RQY)<9Gs86=$+~Gn-_fC;02>yl&wk}^mwp%|phqKAw4YrQ->z#zv zY(RWvto`Iemh8@NHR(q*AI1BY@cz4BQY_);(Y{9TR3BvD(K>X+LC6;XD?m<$; zKxk|)CXDTZ&8VPjK5{m%A+yHY3ZDo*Z#BoVi_uXC<&NgW9z^f6W}$kt^oRpl7Td zMXzVqSMTx9H|dYZ$;bb|6Hh$O zukXE)vAt5yowiy!@(>$w-TILcZUoVBmK|Wl%1k`Ew3Xb>3l+KG{*Tp8QqMkgXbf5u z?)Z{7=WnC7HQYjU>I?Vt+1np*pv;Y8Hn#PH#3uEmOS%n3G-?ZWF!#+bI8@uZJ*w6h zu=dlpS$?1l6muJaxwg&v6A2LbKEQQT1Ounv#XZ*yC(35PQ@WQA|NRQfGD|tv@k|&@ zM5PaB>Lo$jQT0}^WyLCX9@@s1o%saJVf30fossPw7YxLd=h~(XU(jc5X(iTl z#*7P{OXAyU4F;ohRKzMHJT5;WyYw z?>3O31Jk+i{TCE$VbSy+(3=PYX#NHcEPI!a)*Z#y3LWUaS~9;`$R~@oQLVP6ZKZ|I zeR>feYDGb%Xxmr3JAXYT&epLEclA-ee)m5tKkOph3IzO=?p(yX3%66(3@eMF|D`wb zyQiM#fvX#_vZ`z{v&-?Q|EuBkZxNUaslauOKogWnL`VBBQKTjVyF~=OmCR6oO*UWt z`x#zep3i_QW-zF8GSSg7#3v*Y9}`Zhm0LBgKMpTFa@zt<({(tz#Tl2{*tZCMpP;?b%c{Z!w ze~}kIKT7W@Q|J*Ja$)YRsUvWX^elRSs+_$nc>i_Y{CF`33hU4=6fY+@N+Vk_mUi4G$! z=gQ$dh-l4Q5Is0FTv}RM1N*mZ=3ss~A&Fg>IQ=Sm$C$uiCOT~x6NaZ@Q9yN$t9$c=Ik#|iCpsIo3)SJ3;4s72;M}o4)33jb zj#e`mE!b_Q^XgR{ybt*n%4(I~+Tuc-Y932VMqYRrR+AA0p|+%e!m3kvRr64ie~^90 zDuTZk!RPhT6s{3HK*_NK?A&*VQ+;!jBaVUNrZa712aJHp?jXc?GBAS@f;Get1R=yt zZH@Qj-t_=pUz1jP&1m)vr{MEYTUXlMyb51HPJm|3kKBmG zY{6hufB;^P6Sqr0@%w=4N7W#9;6%oDi^ptG2-N4W@U<6s^Xn~?)ViCzPta5kRr$O5 z;Q6QcWS@~srcWfHEufPgp#Inn*6z)tp_$j9V6sNjZ^GqVI=nk!1~8g!SkKVM%M_Z( zs4K2zeBXEsnwR3N4IK2hW8Clo#Gh?Y_BjGW&E$);9{6f1E53_761A3HTeg#1>Lj7t z2(FoR1??S%prIMpflIEMLEm^QXg-P#tYqniyq29W@l$qe6FUyqV-8E_iW_D!FeSVN z0JQo3TI0Zz`x7+1`8g;C2Yn}B$-vYojEdkY-pQNK{)P8f?4qpJP2gMEE;?RkB?nf0 zz;myxz|?Od{Sv;Le%E!a)j{#zszP>c*g&4!Oxlp?Tzgq}>@8(cpW(txFxrtHL*kum@SqdfZjN4x4sFp2K+u$ z9VFnTZ2vkoWS^=J3|J%EG47ga4DS$+#bCr7Vr^sFMKft5uVBWQUPM|9==C|QUa^j& zrS2AwL38G^V|OMM^^n|s1k zcUO^DQj1^H0UfPz9!_g^^_et_u50Mu$$-$QEH0+T6?C56b=8ztH@`Psby1R6ic<{& z3R)n5eu~|qC|0bEtx-f=)YLQq5_I*1{!mONEat|UFaZ^}v;IWpyXZQq-;WaBlS?P` zBidmG-OZ8JAMpIYK4Rzbay+WuMBiz+>q^+R;1&Mz>T2x6u4Zh9u$E;*>qaG-cIKdm zVzFQ~HtI&Nn|im0Z+%M90{GRFpn{z051q=}<~T-OIhBz;lCc<|{@5;-uirsgiwPz= z&YVN+K6so4C4!!#rZa6!hh`vvh@6wa)-;U^5=%JuY2GLK=v$tWGrvwwu~Xg7B8tnr z!6EnlYKjY*+eCW6Lq&NRbuNG7SqFl4{rT{VXq4w=vT@5neAYOIO}~!Iy0oQ@lYz*j z3`UL}O030zt~$wDKaWqA9H2qfPrFxdT`4sUjdvnGT&|P$mC+Q*;7cdaG17*DPSv3m z{Og66S-CeCr?&}!B|3h0Ek`$h!L!f2!SRR@OdXxp3P==`GcCJgE{?8S!S-S=_S6B) zm^y~E76QAi-@Sq*EP+uoXEUN*CL>7vrv6oPXvaajZlf60 zKM|86c&d-_#mmq0`hp#lI=#(!R`|kO~B#$({UCwr|Mq=A_ zW8|m-gqsvJPc>VUZlZQiVKjEcUK4M={{Yjsui7xRr)R3R~4Rb$RNkO&iJK|D0S}OrgQJoxKw}M^e z0m9NpaLtv&Ns0(+Elbj1W=_9^4zX4g-9_exHEiCWON)g}nwzY>d&n-T!V=es@zZB9 zB>DXPG0{)T4mkxdHrk`=ee$m_(vA*oIWCFDG-`CbBBk({S6^4*i%m`!X_YL9EPSGo6=ClBQxzf(fi= z!xqkjQ)p+Bk7s!+v|xM)5LH|Em+=07ldR3~f~AjBm;7j-UQV=yR~X|JDt$frG?sw_Tu>gW7<-cG#c zXhz-mNB;fio5%>iFtJv1crs(JxrHmgImD;i@~O?=&YLg1K!W`NrVi~wl>O8JVS?(Z zqabT5AN~7z-rp@u`svL~=y?v#i0HVhve~;gpCMzqG@GR$s*}A--{<4CIau2d=gxa> zVQ{DDR^KW9at`dsq-aEUI!A|`6c7@i`sgM;nzxwat{BGLb{Drz>)n`1tKf*uVDh!s z@YS}zvAxhmUEWq+ed0lK58lMg2?I!t3B{;+sVX?g$_1aY@}SOT_ddpzLy}MSKo<>F zZ4BCpj_UW|^$8cWNxJwc*uR6L#lwm16ou*J2Ju!O<%_xV*m>MdmoY!#&fBJva$2ge zj>qFe6TK0LRvQhxlLTiDP2#d!@8jBidAzgc5EXkD^WK+}>2~*IQk%{<5kKYom-5-l zeRv#Qn052r%pBRVNvkadM@$Scp>_<^;I1qp^WZ^hhQ<>XsAB)Fy|}_VGo(ix=x&bh z*~hANSJV5dJ~*0Wo+$<+Ar3nhgFI&XRqZ96KtqxK1;Po{+ zxw?waueUseT2%agUt^Fq*z-D|LG$!C1xH3Ny2jW!QtrZ;wVHS5enFCT3K?-W)P`zu z_O55q*ULHTvlF6#UsK83zl8(YsbnO@6BQmDZWgG@X3ery%$U%NK`Bk!%_$g6Rvh*a z48~A;jTl0QaQlgWHy9&GN{qm$DEPG+4sPE=(X{@A$3W?!E$paDVBD~7L|OvDtd~am z_4_=*fo3h8wb_g>*PCiZ34^_R2y;M*Lm z@KAYp9e;c10gmmT!_?9JNQ|_C>Za`YUY2||j}0XeT>I+>nA|xOM>FH{F?@6I3yOReI`kgN;67)c(I8$Q z{%-*|U8m(@91T_Vq6S-y69~A20j9Xq?E7} zYjzLbvMiQ;@&yOIVGN#i4|h!)OtjT-inF8R_xes4j+)=!Vr2v3rNQM7>UUH;ZZBG( z39~8&qPtDv=IfTQ|L+UQtMgH?`)mI6ivo6Bb1j#T=uJX|1+}4yLtB^e`I6nlPq>c< zCiQG4gWCRR)RyFsc_^EaXDVI*cgbOP?k*s4#6&tEc&ak_`ir$((c?PO zn^gxZn5;HJ%qC2s@$~3FkhI9QaiAr#2iM(wH#-iOuyAKShu3_;$IHjke)b6BY^EmQ zpNHd{SF-qSpmO~oZV}f zzhEgzGlvsn@>5fu&#v`LSh~9qpQeELs3yrL3IZ3DVBo|HH{qPuW=z6G{}NgbyL zQdGR220V@WO4I!KeQjO2LTE%dHj4=bKMgs%m_L63JrZZoCB}rSri>%I*0W&g9-Lkc z5E?3qC@ila)m($pU?Y2+=}a8fiBM}e zaWRpEn84|+C-?Avj+IU(I^Iaxk-h9M3eaoBPzFuAfy=($#@wy>_#28?@#d3M7anHL z%uDInJ{GG%qoKN(U8}$3vt|29oNym^%o@%)9OWC*oFr}CEAuYs1&hG8x8B!VAzboMu1;5XS*XKVGO!B(j zG}J94Vw`!~dF#bzh&KP6X`}m)5@iR?OGWN}7QFK@8w-Md zxE?5H@7fLQNYqG5Oe8grP&VoS03ZNKL_t&~j?^L3nLTk4$LH;!POWF};@A0gMLsvq znZeK=?FbJs;A^Pj=#G`l`)no7c9VGM_G?HFHMQvXtG+r8@7Yc9s2EZsEX{&Z<%d@D z&fJw$7?YTM)1Ay5-IG(dE}93gUq1mBp4zEFG1}-fYzBA!WCwYF|D5dFe3s6ApUK18 zGqFqasuZ;@pOqhf#`bC>Jtp1EJ+mh^_ZeD{QMQ;4OrCum3sxUwU4a){!T=^u>WAHM zHs9HY+ZVJ|&;oe9o)(ibG?yEX)(Dbm8i5u-8u9yaHMoMm7oC9LciOx3+Jk)f+8cbe z;UKZT6@2iaN=8aNk)bxsCvXlECbI>5XgJX^aYRRko=1hOLU_BLOuc#rEBCz0zEU?i zTfg9Me@(=A|Ba07)q!Zck$|U;<9k>0?#F8=YV7h5C_BvBja!NHcP240h2;1!j6fiu z;%-j?!@D*~RuSAD4}nJ68VthfEd;4m>Yc8}GZK70?djzV9Y7b6;dzVKv(pzQG#<`||j$ zV~9Q%_{iuWqu(Wr8}c6em*r4hu!Du4hfuAX=|1{$F6owdVdugrCgOXH<+i(SA*=Xh zw&fjS?(+{(TYQw+lZVniE)27x0wLB zY$#>v8_$#Ba5Ax92cqm|G{2jQf-F{k`VQ}}*h{@Cv<8@lCJ*MepZau+HZCb(nFPCuCdMRnc29p}?IM1qF!RM;x z(B8e24IM*5sJWTVTD)&5?|!tFI%^u!@4TBSLpqRJGo z+ufMZs^f9FP6j>z3|2byzm#idEakwfODS=A$l35Q5C7~qTd%o>ONVzS)?q<&m9uC4 z624lUMcU=}F@0Qj9L=Ktf~&!OqENx-_256n;Izf|;p*$JVCDWd*jMJJ`q&2k`RIcj z&z!@wvHeMovVrQRJa<3KKKq3A1rBCB_#{($Ht}tA4>|kxuyWni^q$d|(9^Io2Nr_? zN4q`@>6VP6?d7Maem@>>qrT8IJdL)l@f2EcxjiQWAiCCyd<^(;I$bB}$LDPfuAJk# zg^2>9gha~F zn_rLv6&`8+CQbId_ki?Fu!++6T}3g9)e$Z+aS4)~nkF4GI!m`6y=1_UQF8gr8|Bx} z{72RwDieR8`I~fKg{=DbPo-N-h$tY2kZ|cdXrf#>M-|;yF9()BClfnHiva}1A;WHcRyOCeUBj zX#0p0dq6RW#Stkf?K?<%yJU$9wMkfFce(E2w`5;&y=WKY+8X5Oihs!Dp7CM=L9t8U zS&zv#dCpeX+PGpVJ@lzu-7&0jttLt8d4qge&>$D|@z=?ouV0dDhqM>F*&vG1D#<;^ z$sNCcM&9}KD_OmMqio%_MZQ_HT)zD1O?l>_d*!l$ouz&EQF8AypUBY~we@%FPTBqS zpQL}7BA|%L7Arl*UMo-jXQAvmQYdvUw^S7!lI8FIS*{$=UXr_xk>9+sOiEm*pS-(j zo&0oIJ25K=2CF1w43b+O`iCsqxL-tlf?I&BL7-(OkDo&MlGtlO5VF~h&arOfI%V>JISTDJ}e(D-Xz(jwc>HrOWyu< z^4ep!NYCUr={5OYd4I$4(|KMGh_iT?JazpLiLeCEJ-YvO^8SW=@t(?ys#|h4eJFQN z>Mapgi$t~`E)Tx+rDT=VOF$P1c>@Y(`BTZM@wf2=b@4k3WZ{cSL?rp_%<8%z&U+r2Vj)WZvF#IrX}I4F&SSuP>Jbn@IpM zJL0AP_^EQkO>^X`Nu#9S;7jEfProKF{cegxC_&j{jqV_q-uxSRXZc>K@#<2Ww_AR5 zNt~D?JIf7!ct`dZJ5SKBuDRsk@>gYMpCoBN>_(ZpIaj<*)}Z;EvgM6CrG1!LKoNU< z54rlD-^$~U{z`6|eT)2c-X^Jbd!+K{O8NcO10=kWeiQpnmseLG7q`C!VWfNH=<0vV z}r^tvdP>(D{clH(-QW|O$CzBX7%|g4z>u zURPambj^R{nnCR(ZP2yy%Gw-J^|RAm<+e9vN{={k#C4UQzP3f&XZwx6UbemYGwB*_ z6$Qi?)lVM&WV_U!LR9|x9Qov_>m=ENfI-5N2goDy_KCCEqI7Xr9FV_WKS;tG`8{#u z-LfdNvdP+YsXM+@?it%f>?TDF_Eed4=bvQ$+FepmQ7=B9PwL8ZWaC$F$^A1&Nw>Ze zRj$F|C=zrR+xr*)L^ zw>%*$50;7V)HBfoQnGikJTNIkj35}T((#gC%9`WV5@^n>Qh9WV+&iv|SPj8*XgBsA zS-7{Nx$M&eQeU`DUbu6jB!`+#&<~11EVeL-j)|6Vdx(UDr^tjmo|Vlx)o0Patx}Ne z`{4J|JHaBLh{YZyJ;u$JCtsg0JC77fU4vU{iVn$=xBnzp4M>y3E@R|(Z>*H!djIM2 zs-Z-d{`;5GE5;~*ghXe^o&Q)Xm7c&U|5X=f!4CQ7FK0-n7@L?JsWR!#r)Bx>Tyd$o zh^|U??hbkP$=ju0dYX*7{t;Qcw^&YBNvh<1^Oj8Q9w7!0lRZWTPrXwfd;C$k@0Qte z=Ob@QR+V4e+y5gIx+O?N${=~*wdImudr~e2+?BHN?cd1Y4oT8~+5@umaIKurF$R3~ za$wOvFFh&~(}MSIbfn1WX|v@gcibzleYhd0KRng4=I=L1lFcY61_@2-BiH}@ zVR`(KU&@Ww-!3mM*eMOFCZfA#``Zsl&$tj#P{bOWAyaOBP#%Bu*K*e_H_4-~E|XHX zE}E}i_RfDwCiO}P+H4@8h}jk<3GLFQLwc&jMmWSCnI@C&epbH8u4=}Qnp!X0KmVf) ziZux+5)#o~Zu{59Q*^m1wZ~V;15ylV%9X!U~n>4m93vwk&rU&?lTBsUC86*Gup ziIxfX{!0$k`kU=f*W7Yw`7<)Q(cd##qUEyRd?3fGn%hSKPlc>{<)r+wCG?cvzPCpj z0>O2=O83cM=JXSXF?jzbTcq?FJyqt+xn8DSHcl?NVvaod=SOAgpj0t}7;RB9VB)oM z>&>^w!*8ya25&$lkSnkM>@tZpD}rJYM@$Eqc++p>t*5l;&&waGl8qlcA>%qli`fwLyJ9-@ml?PHLjLsB)AFZBeKH8LX%6;lmb$GqpJ0d#hCmXGj(0Q2L{@5$Bd|Q^3*1E)1QzCm;eS8fgRIj$W=d=&$s1?=NymPTeeT0y{Vtr&2|}f z&s&n$aAx~W4|t?-(>rp_;FLzc&msM0J}m3<>zm06-CHH=U%o^BAA4sVA6Jz=?&prr z#3t^ZCQVXTV1d#tUTj%h7uiL27nTK<1(xr+{gyRkaVaisDFtfO-6ihQ_+&E4OvdG& z-yf4gC8aGbr7idQE1xEvx%Zs+yn4=iawD`uc66M0^n)#8z!%!bm+#&&U)*(e#*n{Z z5{2hJAXe-i+{aFYI@gIuE*mXO>MstE3$@NDqGRJla!Q)W%+3`Bg(JnNu@l6ZGiHh# z@BX!T@uQ8R%jx@CcFwYX#nZnp zF1>2H$d1%~;S17+U6fhe2jp_> zZQI0?H%%6%A-ObVog@D7@ix)#4IUs5-ho!}-eXsajOg&#s>~(gr%!z*+6Vo^yss?W zojb%c_g^H^qjX>9id0$=8J8@w^YTPiT7rl$7)9!+3&azzt`HWlyw7}hmCq4(o>M4{ zL-AJbv^&KITRMgBAkLcX?i353xlc?;j}mI>OP^0487I=S^F(e|nlMKgM0DyXaq|Te_s0AvWNEvsPcy)c7@EvYM?duVr{QVY@9ijR%9*Zgx56)UE z`h8zhzaGl_8f+5_pLEdxiia*P3d;_X&_%?H5o4!{v!_iHr%xI!vJxVM))*^tGLwW+rxr*t;_Q2# z5!H5in0$;?Y=`Cc>k>7~UJ|#Q6SgIi zN+Y6^bH#+y&l1zWe~vh9LaE3~j1K#&YJ(Ut;{mav#dd`LuPl0Yd@6355Y7dwG)9qL za<;hVk1vVk+v1D0BEsEQ zD_;NA4@6$V?IkbMIK&?-ILSx?LniM2Y-q zKN7F5s1*+1-np`J@!~Je6KN@h;<`V)FPdz=BVK1Azf+Wb@?UXYK}1-F4RPYEJD(CY zef#GAgRV~T-XoWYx(G4hrhkhni~F#j zNBz6s{qA>1b$X;_3orcpZ#@0-+st3Qo=&%fPOnF=*P{_WIvdN`vT+^jw^Sl0k6~o0 z=@46K2iW@Ad%XPeE4=l=$82jFK+@{bYb4y37Ah+%sBRe~u^^j-2t6t!qLMO6jttV( z+=A5$T8$bJ3WnS3I0tdq`?2*|>F#W!p{AUzn>JBa-$%;mDHJ7}zNjZjY9q~8JA?)EUyvzz+*W*nMyF1+PF9{$N?M0Rdq>CPU~#?9dRTW;mL%g$$VX(pleHT>`I z|K#O&7f{yWK&928*Q*%pZlSt-7iE=AcnpbTXY9KRNu@E7UNo9f`AMi;RxGwb0zyKP zgLvF-e6oT3(`It_gTLm1TP~&`DdMOq{@(U2y!*ymy!+ley!p<2cJ1y*XmsfHYCN`1 zYRh&}wrdAfEjFUki%5zb=G%gPIx07?rO}4k7|YlTZ{(KCrjmFJk2#{&lbln`_z9y) zi`L-m>!Gc+fwHX|S-)-#E0=xF;)M(O?6c2Uuy`4pt9wYAcq#Wi^dNWq@Eo$fatD#0 z?)r^rtm`Ud*oqmyW)E!53{`} z=xOEicjmIK!+|j5aoLU2k?Pklciv|#S-g;s=gncxyoFTQV!8b8M|kL-YZ#vybHEXB zwIQ0^l5vz~oAC_vVjXZJBO!eS{xhJ&lZ^on8LEa^8FPzdZfI zTP)dJk4tFK>$S*U8;w?$j#$tsamm`RlO$md56dZ<|PHlJzHAUU=)E=ajLND?wr^O-6Y#f`N`Es;erf z?{E;CJ_2Ls8eV+yMP7aF9hPiqz>NmIUPI7jrLn4#%IaEN5jhm3Mx$0qs5K@s3vw~} zEHty zR(7mgK`$okE#++5w3S^Il~k2&XW9Jsczf=0dC$m74{;)&Uq_S;<$6(3KaqnfIg zUc8=unkvg_8ZdC?wfAu283sOo^;w>N@okoEZNxn!zadW_jg?{fZS7Z)JtC9HJzF#- z4N(bcq{d2gHn-5{^x^k7Xm4r7smtKJ8}H%%pWMjw(kPmDZl|H!fzRW>>e6xg4{qhE z>7~T#)#%Jgm}M)K)lCez-B@?mQc>GUbm>Lheb3EI%85aZgw`0xh_MqWOpU;0?ZGza zLm(jseYo8oFeEa0=FR--v0rn`MH5Ll;H2AOSku8SHg4QTUr5bB(=Ik|-pcm!Dyqx3 zv0}j--uhrEuB`KT@X-gj>U;SZ4|&O~gih1Pnom|>jhw_?58cM-c>O_kMLMWiJBR;0 z`yy}7T}FMM7p+c@4i8<;4b<1SFc69+J4eg*IkR}-#aH=o!Fsw}pwsJ6$!^-~tEsK2 z!(q%K&E#X_+-Larv#+pVeJ%EY3cX%W$k|IvV?E7n-KgX9$x1MyjZCIw)JQTDO!x*m zsoK7Y_3PKMa@k@&oj;E`bLO&mT@_OHS={^ZL)>`5L=q#08(#@SY#PO5Cs3N3glubJ z%Z4?qTfd(5>(;Pr;m5rH!92Eg89DFP`+4}D>zS0Fu$L_k(!OH}uRi}QufF*{%XYL9 zRO`^|Gz9IPR996|QPGN&l!mHp4KF|U60_cTk5%Pu_#_Q_y^i2u4-M7j)HJpuN9T~A z8i^#S&_^XxJbpX{Nd_F&9{Qa=grq`35JE%Zh|{_0rw{YU&#q%^#=bj|kHjCdlrLxQ zk_}i~5~-ul_`3o1baKWd1G5X_?q>Ujb*$U4 zkqztCux!zM-hcliHnqr{dBgoYa^Fo%ElA?PPH`$C%(3WvJycb<;Bb4ev{X~xU?t-gzi*D^ZANU*1p#T%PZ>tx!>HnVZdHp(ihso1fe1#{lzorRTTo_jkFJ@6Bz7bkwL z3&K@uJqbCZm^`tR^k_BRb=z36YAqW#Y+%i*Wz3)RJ|8XKgfHz(e){ku+sEV1K(p zmKH2dA3?XB&fUAQYtuRJ`n&nXJ=b&EsAM{~ZJ?&dNzgMuuT#hPi*DeuF?yE2_Y65uYJ~a8X>4 zi#h7>?q1PEC6SYE#?jSEyJe7I&_i!$8(j`9W6ryQ`|iJ+nNv%VI<~T@&O#vQq0j0f zdBR0pckTJ4#Tby(TB76QaJN-b+h)V->Z88AoK|NvGj6_zTQ8bOtU&{k7IR`Of$n;$ zTP%27c3NsGXs}3(yYx2hzW!V?qBKZq9r3wi7+0KyK47QEI)E<(l92IvTzCU2QcBO^ zwrqVlB=-5Wb9ozOj-+#w_x#M3C=jrTGd(>LB zYOSid=X@jD+3^LT7A?+`2;g3AHSWe;6>rwo?<&t9i{{Ses?2?IY%40Qjg<1mB}0A^ zy5=8_m0RAtWJvOdU1-l*m$uhtaOTF`P^{v(;i=II)JrShFQ+g3z34mpt)#0ht;rl5 z8*tB7E5w)GkhuNt0F=6ye@HC4wJV+6A1|>&=*z3AX{ctLYA|I@R%!E@Bob@*6vQ|1f0wZRASsfckjkq-$+cQQXKRtnvql zhTKkQI6T*uqiNuky|VQl)U2LooL3EI>4@LV(ai1%CCwyf^4kKlPT&KhuImn$j~80` zte?T`oG^ZIX=Oo>B=twK#P;En*Xt^_*IThU`|7)uo6lmu;>@KF9vKJWLreCN++fB$0g z*vPn&!%Su$Gb|fVA%P6R^ACZ=Oh?O)2>J707o-pO)$MOBxR9PTCE7Aald*L>Bc2~& zwoJW2Y{Qr-D1{QG^C-z#BY&OAr59gLnzfu?%mLCU*WOOvq2gunhcKS+A%;R;*hdKp zU-Ow?MX7xb8G-Lay6auXrt{oSr*yL#0ppQ(f*@%ZD8JVmsmV_(7Q|qL+SgTb{_8zk zoZL@yspzDI{zFB>z22FgK|a?E72CVQg|j&uQ7zrm%G#=u`V~bPi|0jfpcOef4S_(g z^H6q0-sbsQS5C7n*YAK$JPrDnG-RQysZ$+d)!O#V$bbNtqd0{!wC^%15zT~T`{fP?on6pGz%Q>Uj?g^LAZ8e@dWHy&W zLY49=5x0+_>NP`uh)L$R@0=QnrjYKWuJw zUP*L3y!~GGgseC7|F8fooI)XM=8fJ52bo@XAKrG>O&|e3gW~>1s>}s5DpIa=qjhM5 zz*imv-#Z3DeZyE)q%(B)J_S{K%!5vk`tA-wF%#P49vwliPbhn4 z4ZG6BQy!C$lZsIn2Gd-ve=lf*=cPTkrPJV)Z=v}NjE4j^z4vN1(?*_+u};|my{xq2rx zZ8V*$R6i~r@D`HtCAaSmeTEPu@E36Na5r@ou{l}YaGPOjcP@&~>Dcr#BX3VFbDg`7 z2Ar1D1=;5st30+A#V0CV`S+^~E$ z=&}v`b4!x5#$c)=9$&lFgCjqXHxv2LPcob#CtJun%kc9#GH&XbRg+2-jJI{_DEq5> zw7M>EG-AymgqFdl+XncRsk$&5fsq-r(UhY3Gw`SOmEUMggMwK+QMZEZ=$44OJ0@oH zrg)iUG9Gp?9-NpSG)3`93SqfL!HQ__L}G~wDO_+|M%pf0`(9?^Ova~6kAGgfKT4yF-hB%>Prky|`$$~mc_LqxkxWl4zxgZ2g=2DU4o zESNwy()OF-0mImaczNR9sfoI-ab04NBH?C&)1@Ke^iK;=e_v_sp8O+sRQQGLXTWW5 zSEJW=0Y=isS0e9IjV<37>(8fKzF&2Xq@Ycf9gn}eW^tLzRC6dL zHO~VWD2FEH&89zkR?yjLdEEA{NsSZ#vu)81jSQ)oI|SiT`=k+h_bw8N+kv5y{aAhu zPzo@@b2$&KHE3bD{^(4k(9jAs6)ln<1+vb znqb(?IpU}^e zZXV$XmveeoQm7ZfhgQeB^{<+Ou8wGjvu_F@DNL4_zip5zjo)$k%EBhoAa|#fkCD>N z(G)xwpSfBMzjQsgIn3mHQZtYEB@;@M?fGb}TD@6@OO>_sM2!QmXy7dFVWeez?5e%J z4&}SrD&q^R`Fs7}v>osWCGKi3)Hoke=A}|IR0k{OQO(K#X!i5|rMs{`FeewM1(mfa z0A)9-|j|%Y38j&ny8T%M_D;-hH$MrXt zV%W|zvbs)Cr!f8CogIaq5wqnD4+(D(Cp=BMkr#YQY;q2lgRHLWJC?Qnz_)IqE42hq{uo8RNnhJA6J`{QAfuY8<}c zIr2N{#O+=j2G};j|H7?;zPO|Pd!!;tdYXi@G!4|MHDGvzKuwWAMWYLXz10JC9%l}g zI26MFUW@+zaSir(vc&R0MJY*2r~lfNKW^W2{h5|)`wdxi`%9ql(~V;+BGk$Xwcozg zp0h0pmN&Umm!41w}VMc(`O6g6=#m!Yv4&MHyN|pW^7h zVP+j6Yqq(Gt1k6%ZLD5q-Ge2}W z%xiP1`y1&Ml*v+)HGv@g`AQl%aeQ_dq64Dy^42=yR?n;nBnnJkf5>`iIPvjNw}-Wx zSBQaHk-V)i9@6Z#?!-{?JyPkjZ~i!@sxDsILe=5+CS|vx}_i=iJvN+c<#w!bXHovF1j6ru5O0LFN*1?jm_mpt_BPyurNy+Q1vO0T0&()r6tCuh~PuYOoqSsw^m@C=%iv0+GNxd9AHp*K9uZXW$TQs`pZU&9gek>P(OZ#$PjYO6o5 zE%#etv{`?h8?s=v+0c^QFG0mN7pD;A=B3r@b`f~+51E`FlOr^gk?hZOrQZM6(xT&G zw=OB48ydB*H!O#vXsp4S9MNsp0LwhPG+F!K8fhWPZp7g;rYFB^jOMuX)Dk*2Etmmm zesdhCBf72s07h`T#K=$+C4=6BOnKP zGAfROk0F4G<<7*q(&TM9kKL^^N`}ttJ#22@vJ9mV(WLd>#I+;xVM*PFAp({S1j2y< zgh|YTnCoOUbku=A zyo}cUn=_)0wa|8_V9Uc9tM`Q8KqoS%_u`;_lfYCM(*JKq4)u0>Y|f#BsPai15noP! z7k$bkrCJ#1(A(&vs#(p`2Z+a#P?b2>YJ(_o<#Amib|G5 z-0)a!b8hlN%$U3^$udE_bK8`%3(8SYdTk3O?L@WJNeil4E^v&dsL2ab`(!MOJbr+` zu}`BlV|&8`YNqaLBu#BRw9apQlN^S6y*_kpvpekGg7r>Hg|#6lZgCL8zJ_3MZY6X{ z@CPyKPes+V?7?Sl1f7#0?)Ki++{b0;DqFl0-0OnqC<+d`{32BEl2QN~zO$Kqez!i~ z%hiDAb(K4Go!}FW<$g|NtE%X*fYG1%t3hJ!-&46ID#dK#DETQNTSCH)j>L?H$KL{F zYS@et6v-H6ObV)Dau)`IPc=fvegJL!*zuY%6qGpZO_mpC>hsgtP z2lpQT-FP|q^0(!A_;jvWU;QR2&7Tn33JA89nKUH#rUz}M^={{7g=HzH!yW`+_FAq*$(QgGq^4(+7Xg6}krW{?b=29>=o)N(T!Ok>(fd2lkLJ z*JCW01PT^%L6=SgC6C4@4ZF85$cqAE4aEy5(Fw9Rlq+ZnxE<@Bs)oAK_Nr8&V3>y` zzntE4(j`^&77YZ;NcLT4myG{#$O+mc&e+BzP|=Xu{MBf2>616~qd1IrM-#xrXkDpp)kYjQb9q>m^|2a z%(PY$?ibp$l3s6b31$Bgk`|&e98W{R#qr3z7wzvaUtSY=X#gWaTF4@vnv*Oun9P2S z>|O8SImd<5qbZM%Dr@7?Ccm(AP!D}_dTKn2?*QRj^7VDr86E&?KAKwpV-p&>bifz9 z9|V-J7PWLuv>cYe#QT zDr|>gDqtN>&Pxso$SMlI<^#Ri@D5zjJf5`7RG%HM*ZVQ^|61^gX()cLk%zACO=-77Ga_GeB(4LkzwP!g4;P$aED7iR zCLk8RF`zWbZBlMC6j)uVpe$Dn;*uKIdD%ARAWo{VqyZ`@!vUq^sQP~!_DI3LE70Y2 zKZc$Kf_!1Z`Wlocm`gbG$3$EV18xjX{36{_Wb|3@!>zShMO9tIzZ$A;T-?WirC2H+ zz{U}L?C+-%OyFjE(e&^7B{G@|u&5*OU+%sv&W8j`4Bfg!!mApubU5w1OpcbDENW{C z3pqzf*)^TLG8D!cPhVPPAG{b#e|R)HIVs&{Ku`T45g;PSlU+GTU|frU+GpHs;>vvk zg_QA@NXkQjHMXGWsVLL~aoMX<$brK`Rf1A}XIg)6;^`e3?dc{yLXm=k?>_30CZ!K} zX@TqT(R!HFH#fBEEwC3ooIg4%b_t7%6N_MdYPfpCgP6#>bZg3 zLl}4@CC7k&Qp{AdVYlJ%y^l*WHv=rvp5!X`yHNXaMUnWOWa!+*`>!Yq(i92@q0lzr zA+IN%EbzfDstK16NQDc^S$~!e|v#0tHgr)LokdaW7OybGRLyC zpob$IZ*Rf~066g5GoWGAB3ZcJMs1k8eyZCb+B^iflA4*RiOn#fxIE9RYZL5WDXN4e zB}eY-s#MNdEdqVVa^p@5Oli$<)x`El%tRaF2vSfZ+ZB5BdR@HitrL>FU|(W6I(uuk z?%s@;qv!#PIibG_2VQdh{rX8nyl3=TXxZ^Ns!R*MNoci24r`&Lh)MAe8>0Mf__vLG zGm3A&@i}Sx?x%y5lPVk)P9M4LJKDHvsF3*FC|c;0XkiUR&i`iXJ;RG4^%7vH4d8jA z;{V!Jog7W>&yRl zwGz5e-yir9%5J)|_h>S6|04P#YsZ8oXc6T5$r{MK$pixd8F_wpeS3SrPcR+;%L8P_ zSH6hfPhBekauAS zco`CU-ZpduNm#mU33TY!XatgXm=FteD*B-;Bc|y1W^KQfGV<{I`Whwg?{H<@48BB@ zM8ZbwGcC_?S=&B6nkSyVh2@tuJKi>Sei2K1e&+UWx|7mSE=5z6HRNxtrSm@=1vaSr zVkPq5xK2+dz#fr3?(>Af8luTR9|bi=PUUwGe!XqOPe1pzzkkl(7st$ncV+5b)dWgB z8_+#Sm#LcZ0Id{LE^uD2Bhy!QbQ>moS3^{3YCkf-)15Y6!inp6;|QjbY7xg^rA6Hb zz2oZUx8DvWdk~2!?U=+M4`x~XG9Y3Bie2qVFy%vJkqKK{VJoHlafAYxrh}$Q6VKy% z+JCos(%oVXO9Ec|zVQk91`7%Dd>?r>yq%#dL-8FYbU+o4%bs3h)2ug*{`^;i1b_o+ z$Df17D}9wzkn?+av0dad zvyQEXO5xlo(hF|ndv5oaUX4igC*`5k+6r_f2PiUq31p=v7_peP-oO-Jm7=d9F6)JF)Wp8T6H)Bk!#6Y%+dsJHC3O|R7dwggI*%qT!+ zYg#uQC(QAKirdJK*AZ`j0?#}WLK%&o2?Ck2l7XZN#}OvMwx5VIPxT8FQk)qhbO3IR z4x!_QhL5|%iiO0TvV{?pz#>dwQhn>yetPeFhNgz}`};vPw*VF8M#^2afgo9mwWdLT z)lAi#tvZ4I;c0;jL*?v`uxvoOOz0_h&L}!rnVQ|7`b54&;A7? zV#CkUW@k!DO57ptS<~Sv}xhyV^D#x&(#(C`Bpt$={VFdYvsu#!R;-RhCw6Z$Nj|y)6b%xmOGQYktwi zO_CoRK1MG7N17CBu-0->LT7`KDqDp4r3jKH7ZKTk<&IWNf!#<5SmX7%e(h$jc}e2J zO4uJhDF#MGi`S`NEwnOVT<_qy&2BIC%_nYIhE|8^1~O*@hg_d8J0?Z_=GL`FSq*#d zHZt==QN--=6>9#8j+#{PyU_OJhxCtia_V2`dhL~Rt^Vc{_7Q3RaofhteH@yRGF`y<&wAFbzC_oNnt5NRadNu11UA-Q#i{ns%}#vlSUv5@ zU-$ha6NzPMJG1Y0UDYZiZ;z3g$oOst*HP)f4PMvrj$Oj9-zKsIY#Aw7R+uT-k}F_8 z44x3fbVr^A6wVsz44%(b0MAh?FCtGmh_f-UTqGR4`Ln zvKa4bVJ$lp)p_6szM1pc&U8lMS}K%~a3H0M!~#S?Vi0x$3~BaTqmOGuDO7s>c~2XI z*4fH2kbauUV$XU=8^OO@5PdGV^;ic%fc#5mBhN^tTMRI4kH}1=sMPN4O@oA1n9c!0 zPC2qD|2`2CFZk|)kbdb-ZZS0`ukgU$YtAXoz_$Ee$x!&H$=qp2$L^F}_SA;hYd#T_ z@`Y$26mUp`#P69S<;T`gP*CBf@_hpZ3+O%kI*?90gkB%uapfJ&n9mT5(E5Uq;5_>S z4xV}cpvXiFhu{tTF0}>e&crGfjfJGqfMs^IKF3Imc|Tl4w^qyMb4PL5(r>cqv{!7* zm+%7*FJs*G-2S*xrXQjbn~PT`!@hL45pa5}oSW?(=DCHZv|lg#wjK04i`!x#D}iFl zOnmsrb=U0$RhattkFh6dlw^In?VyQ$K z!kMxVp>N*Q2M_Z8)I5EJ5dY)>PnFUBQQjh-dc*BwU9;YAN!L1elmaGVxTk;>a0{2c zwlD^a8zLYKS5RJr2bmPA=yUy3i)y1Ys_nS0(?pPyN~<|fJ|oUdY3dy?E}nvR+-dyb z6B%?uXI*FckfOM{dI-^Xb!z%Sbyd&)1uxT>5*^QBI-lp2*=j>p6vb&piUB}DK)ZgM zl{{j;JACv0_jo5gpI3kjCJ#wI{fXM^y)mWA)H{=-KehPlBkHvIPRZImOAZysoz~BR zYAY$>J{diWbB~Ou;}fqeeu`3*CW)Gw`iEaeM!|^4$|YU~gSNOiw|3;n)>Mw7ppnEO z1%^2Q&6d-2u`(?rx}Pfqpzs=iAOg*sTUkqQIuY?;S8Z1^VKQ`?!6Gi1nv^4blLaS- z!xfQ|EU{lQVyc7&wXNTLwl&rNv}Jh`p})w^zHo#T5m-2nlAa_Ipuj6lQ{49cNX#n8 zarjR}VO0L^aj5HK8jThY-|EaAC2gR0UUD!sJy%vog%^FMGcJ2U3e|r;S`XHCfm6`u zO`lVZbGEG$+mZSgt=bxFO5@$Av^>Ca{zXsVj$pnxjia>S+Wy~#t=ZgL!HyT}!%)wpOtZW z-h^eIV!DDrCA74*r`BigsPMf!I;sRbP~Ih!lq@GH#LlZ}^l?;wvE+8nEJtqJyIwB+ zTDTsI5@sydCac?CZ*eio#eZH)w-i_gEx+1~w>X+?4imDsxEnXG!c`%FYVPE0!^!7p zbR4#-uw*D6fcQz;U||R-mDo6Lq%rs&q`RWD4GA)ykATr9yTGYH_z zj8hai{l0;T5wV9soq|w9h1!3FXEu84Fbu~kfp#DBc+$LA#8h#3ckL* z;@m?c_fV@r@2a3h%~tfVYTdTK)~`K6!VeVon2Gb*fcPo$`bOv#3Z6$3^Y<^#{kTQY)$sX5z>AnRH9U?f<`3D4d zH<+3Z`CPpP8R(91`2YXy_e3iN78U%jcAY0qlLA^{ z|C{+yDgW=|Z7c691qmx@K#T71#Y*CWgVOx}Z{Y({qoiGoR9gjc0O^VcRB^l};QY=< zASmBo+@mBxrId(T#`IUHk`g9bVuRUr+OF1{V!JcsY1w~xnGv$oI|E9Om~Gwxl1Jr6 zi_~vtm_7Ie?#bd|;9ehKVb7<%scJg+&=u$_xrsJH))*g&IQI$?y zrbRCS`ND}>fDs1n8RD7W!(%05?w`Er@CkV|f4T`bDJ*msw~wc4ef5vmWk$V*;6;#l zyF7ush3KL|Pa(;%>%#Ver1>i2sjlyJ6`3>>wfTJ5oY`P*f`}}<2sKI{F}GV>;rubn zanB|WImZ3Y=~ZE^3%0s@C+>A+^aQn0e+e8$$w(nzuxpeHwQ%3ni2YTeg(l{H-k*|p z?k1V7yEM|{7L{&u#9aLDxc4mt%0wVss|^WLDWQffIgdFjN_lDu z9vROllkh(CnwVUhGMl_PA{J?37>JZZ;B8y>w?vNa3)M z;LrR>yr;BQ4?}F$yIN>u36A_VJZsp_trz>DJo!q?CdPf2WWN=m$hz8!OeaeY^Zzur z8ps)xT+YI8S(;zKI5wDBes2MTx%`MlrX^HnVHqsJa{4@2VeMw$VXpec`j`MO7;q&p z&X>o1j>Hf45_=Q}q~C?{{EGV?Gbv?IkLX>gdyA$xvl_GcbkJDUELmh?0wne%_5A!t zk7jxwbDqEWY_0u)q{#mXILrCKkn7Cu6@1|>Juzh`IU%^KjLrxf zQn{P!YIpnL(t!%cj-i3L5d#yofN`$VeHv=N>Wn`iyr4g#?n;f($!)q_jTO1i`_g-j z73+!e2v;Gxo9FFVAkj`V7uW8xh0Dd&=LoSwR?KPvB+h?{Xdto?>^wsOugdec!xFqk z?JCe*71{YnV0tk+T>K~NPHx}&_T+ft)^6xQv(c=vEv>1gJidWE29zq}Q6W{pP^qb^ zk}P;@X+mk^_P>FpDJ$}(d(O1_#+J|m&YYsgx`R0?A|pxV{yFT&{aoHOS0ey*#*}_eq>ojR$8v=Ivthro=zbrsBy1lKzgnYKg$frPD zp7>5;{}5vTUX+FC!>fp{dyi)2PFr-kOa{|~4hD8zcfFStF5KK=3~7ZtuT8Omh$rv+ z9-+ocO^Ge-)|-4w;)8pPR8~&IZsFgpnRjEljapjEv2)+-)z@hpFCk39_n5lksfa${ zR2zzWi>T>gyL+L6>+hxW^8K zAl|@-YRv|V0W4a4A7JM{u%C>0@}h$`&2LiLfaF=KL)I5x%R-J{mY^z^Mh}~=41sd)Jk?m!#&edr#~v_Xv%_Y)GI{;h$UhTbV2}D7>&&NZ z>#m2X)V(74`QllU$JpxjBEFZgImn?ZQ<_s9jS=g%S=Uq-we-aIfMqK!Dd6!D{75zZ zm!`3vl5@P6{KGpTGD@B<`|eV7dFkPIIgu5RXvlm;Zy?P&m4=2ec(nkh9$mrhgRui| zqQdTc7*9>L)fFGHlFEzJfC5nV}oUBf9JMPMU2M$!xMkL3VpWE4kHtP!FcB7P8dCqeLi3Bb#>N8yWbh?Hp3mcN=|Ew z6UO@|vk#i5Uw1cl<8am*J$8QA(p5uQ}rv4odv#Qet9_xV4ex?wy#jR>{#>cOXO>*bL`kzPD$0V`OeXD z+O3J`_QH%M1)q@+GGi#rYcc&RYi*BfcxC(rhUv1(Y2yp=<7ZLal;+Enfh}U==POQ~ z1DErP;pe2h>b#ZV7iF*aJ(FKYW6qFckLXxaBTlv;<{gboh0>#zme3K;=Y8gpmiCG) z5Pd3&hK)P$P>wMx0J`+WTD-zfOXuuwg9S}27K8Cp89ZLA->?uLZ~dddG(@YuxWO;5 zbT@WZKHyT(k@tP#6`KE;>HCQ#b~2jPzxZCkX+uLp$)&ZIt`z>5#u02b8r4<10NOgS z{Nh{=;_mS-_w@6%rsPvj{H+4Kfr6)?snwCAjBXs@16XilWZ2zwL9$;9YA8cf}a*235Eg%c#-`k{%(dfxwwi-Mwat>{$N>g%2jGo zk{JEh+XET74jL5B>$X`+o0W#sjbF=~W zd82>6N&EdX#>99QD5^3NZTrRKsdwRC`-T^HDs(;NW|J9^!%5em zhjZ~$bP)8Y?qfm}nZNW7wYOU&v+Wt&cyB6o;k(7Y9xz3pDNLZl;hGNSeQ7g7gEWzO zKp%YF|1e{5dHpbnXRw@zdps1D>5KS z0NvVtR|%F9XUb3?BN3l5tGkB_F5)g@?Yl4TAEHReQNPs~QIHLIH2{&B;NzG#Lc+nGneDKY3c1GAyj;1E(EL+p=mlixZ`YFQPC%r}hCLV`CeWPLQH% z-o-{krCHrC1GuQ=l2Kqvx)OzJD_4wPjEV-G7k9gW#7Z5G;`n-{?u`R_uA6RO%IU0I zs-t3*_MF0nRnU`pL;5j7RdD#TZL%5?A1`#RSj?FVsUsm2-K~N7Ubu?<93jLHP|}kO zWz-LR8cP}yD%@m|z+xlQd%I9LLCH z06Qr#`d2|`roL@UaV~4#Tz$+B|GZ6H5J+sCZe3d7gWe@>UiyR4 zL6*YS!z=6c11>A^>_aN#pV4<&O;6^-U(qF!_`TdB)RoOyla*;02k6+P#nvR(HStN) zX`?Ss@6n5mSA&*vg>5B4ur)3npaaPr-M=-b#3jZn&_3jfdvUwR=+BGIVUpEUw1VM9 z&g9XW41*pq_6^7{9oj5!Wu#I{l8+JL7J11l!%Iv2 z)~OcF`E;=Y`a5!nnqE?+fz09N+NBhQPQ1PEBXoN6C#3C8>@<>yQ79h|wyU$5Q#|O6 z%!IA7q-M|{md&{h#ZT+{x=xnA)E?kW{Veexbx3Sx3_OrL^RNG~RtvSf~U)bT{ z+8e$Gq=rqc*A5&?x8 zwn|cEtYz$pz~Ts^mDZyu!U6hh_;{VffMc9r0BI|P;o@F;nPRcJr7-OHZ#4>$3a(9w z423F{EXNPxgIUT@e%yKv?=6%WOxly~*osCr5jV^~UQqloZMC~9iB!kjBv-K;o7SLG z!ektUPk7x}aS^lsY_5ClWtThWc?y-1N6@7Nl_VA#O zC1u(0y^?eJdW?_2#~!|}D%Lljy!8b!A5uV=Ff1G?9AQbd8^#HVyaZ}J_IWW9tqGVK z3Oxz%sv8ED0!4@Sr28p%{q6O$4K><`(DKkV;2c!)2y>hFzHRzmG(YlCT@rDS?|Lgr|09M5&yK#JXxB0#n zb&#~#4-1b51>WCD{*SVaGA^FyZ_A;q-c4>|K)uMbqCaZ?FWR9>2DW*aiqcRdCG- z34caLlF;K(G%y(o7cRqRt0A=m-DK&=Llw2?{Vt(zzob=1%}_ov-%bJ{E}v=IVMDH0 z6cORTKN~{f!Mn(OjfLf?+GL>m%A_7&(7D|OkUO?X_2#q^W6KptikET4Mv$eVVw=*j zFK>qfi;D^whw9_2uQz+Dtys7GrOq5L|7Zku;h%V3AW{5dBGEc!O*9$0bqCU0JzsSt z-d;Iy3N-9-{cB`uC9gL4YW?Iyn4&Eq{=}MCA(l{1y%F+tsYW%|d^BDiCwA&?!fokH zc3-(%)95Xxp&@fs_XT`Di{Qexc9)~GGZ8zWln#(;E@p?+TxJYx+e7QY0 z@r4!Cbr9D71rw8A+Ji^I@i4ot!+4>|I960Wv|2-V^Dy05G5>z_leSbMJZ<3Yx7~wa zTE$fFd-qW%!#Zd%hRSJJb5qpu>WgiG&-`*WJIS;XN~4wF?-guXh1bVHI}1l9T1TAE z`pA&*WZY0g#^VF3v*d$_ArkSEcQDS^X7KERA-rsOi`3Ih!O>qZet|2W7#PovYhN!+ z-tde&ID8JUqWl+$(>b;azdgTr(ztN1&yu#F&7~=K6@P|u_yLHH@xjBNFOj}sEx011>_PhhV zZVOW8_+sE$a8{?s%BO30I8?-;5Bu4dqa99cEtvHYGHIn2L@r6V;vfpYQy@FRR}uaVt@>zx1cnpFRTN6p0wdx%#WPl1a+J0BoGDN@=|$cytckSlK!11TBO! z;Z~QZm}>G>Qcvg}5i9VXA!WQd=wC=ggP&PRd%rxH@a}%}qey^LTCA>uaqyT`SW6|i zc6++h_lty455`^+ueFal)g@9PHpEqlbSpJn5@$Clr4i~uw+mQeE-B!UCoC%(7Kqcn znCO9GwN1G~Y~&Bb>>p{!?lRPHTump6Zj>o39?>G7LPtGIug=l0EeKHny#=}3RP$(W za>B2Ec=pP8k3jbrQMna`YiD<>3#MwMC3Eu%Th1mMNpTaNjjj*We`nCu4e%7{53{+t zt#lzk1Q;%2$QNHRQp|5!1y$9O67J-3oZu)Vu6H890%U@cqUAH2#lp&3gzPbzQJUId z5v4vZZUX1cPSU?CgtvUKWt`Qxl*OpyN}4}n220e`ayL-9j9@z= zy+0D7kwD)0W3^bs%0vjILyT~|iK>!|HB>1Llwk-O>xO_v!i_>QfQS8Nr}2G~l6=vf zVsyMF*i$ah?J?!UJim({(jO=ZJFZ_5O?i4vcOgjp&y+8A4%_>tw4Yt zoxUqagNmv#JVB5z89!Jx7@|2(n+p%FM6(X}tGoDBPxowPja0R!?i#(1+7@@HdhGT{ ztt`~6uK6t23JDV%rivw4h&b3)kkNCia?^VcXx5P)%vXUlnMH=Zz&2@yigDS2p%M)C z=#N>`S(HJ@&Q>`M6wYcW$vm`DZ*OW^1~GMe2saIyt`I(Q##*|N?oE=n@IPR=O9=9+ zh%t9AN=hnlrOH8_j?5k6?h*N+xE9&S{CN~8kbEafDSjAqTVJrgb!oziPzd8PW~JqY z(u@%#8hu80vvp?zD?(*d$b~AgNfqD1vno?ym2=kzRl7(Ef}pK%w1SWueyR zC>f@OT?uK&IU)hX?4e3JeD=-D#B}x`MUwi-sWHGVV+{S|@-)bE&ghcU{V-3U(q{)y zMRw|s7aHl_^$vOogqr&8W(#}xiLGg@sejpoNbJV7(gd*c;Ev~0%b8+`~lA>BWK|rC`2)Gl~j{LzGp5dTRw=-h=`%!pWg62 zsZ0bYuM{%#VGm)ZS5!JisPCHf9y;XhoPpSgHHU|+8v9FDih9V9D=V|9E7ir$|0M(^ zF^hY|i|ZyDR%CMRjYD>8gvV^Sl~^1WYnTRyb|Q=Rlg)ClVh|;`L`i^uqjMYgQPWr! zvHeVbL<21_2`gR4Q3|pk3-%?o!m>hK%Ge7FbgGmN(h5_hv#XtVtvW=Tw1bl3L5!f_ z?g0|kN~J76IBDFW0!?mc#LL7-qTBUbSz`spmhU^SIyQxim56K5uLDaWw^aXrK~bi{NxBQTu&T@z9ETWpi;))Mt{w93 zm+1{R|6;|VaDa_Hi4~kHbaJ&?o#Fg`k4G_nw~+iJC)B1H>e8OCSh05twzCv^gNDc+ z3%#kz2g9t4jG-Dfo)`Z|cL>T-H(!zH#||KMgR>h5a#$)VuR4Q^ib}!5RqUs}A89Bq zuQ?`boxY?^0Hz$PqT9(9%qF?m&ho7czRuLFbsF`Qf{2SG*eQpGSD2ebZ@?z%`N4g4 zUBhqfDdy9q3dV6!9Sg83{2XJf{cBiQ2{$_~w`*I7A;Q`fq5w`UImu)7=FzdypwbJ0 zW5QENF;}P*CIeB)N_at3e`e-2R_wys`cY}3gdGoY!HKCs5%YEc3IoaZKe+l=xDi`Suza$jGq08hQ#!j%do z?WE|mCVDmY6H?b35iKGp8AARG%Le4K?0D zUv%f>SJ1vCRnzgAvYr}L&1y_l7yPlj#y8I6^rg`$xzNAe2fE})_9*{oMg7Il5wHXr z;OPoj)=Xtr)S}Pt*((5#o&X@>CdzCr;l@E=V?wCVa;9HNSxMn<6}a3>}hO|erUlnI+!V&tx}F3 zLVnj54N{p}pNVT&OFcNxNhLE6Jj&RQf7pMKwrU#;2k|Joq0?V*+cJD0FdukF1=dJg zJRQZ|uOo}apO-cqW;u&HsK2kMHY&{(gb%Lb6Ox&t;^V?IVGs2-!PsG;yPZ;|9a)l* zm*X_cFi^Wk7&>vAbV!iQP9VBe9+&%xgVsJ?n)>3vtWAz&5Rfc9*O9O3!Of@O{|o8A(nvwXz$t0-Cwk!i z_eI15;7qNkk{P|IKjEyNe%Ki{BT2c4uvQ)EG3Ft>xjBWJ#I0BxHj<_iJXLsVzfr!4J6$7)`+jlu2H6JkFy+MYvae*2ku#tVrMu7K zz>RQz9%B%Oj?R8-+Z^;4TB6oB<&dE}GiK)`FaHhCyOvm5$sj4|gQsUk#z*OPFYZ(! zL@@OAo$UbRPaJGT!_ay7sQrW}9Bu-Zt-$y7ju2a%2&A~7Q*s(Bhgs6MHS-PXN-^yG zzHLB;y(D54-MCU6r?7T%xJ`QV!Bsl-^t5cG&&HFoQ$o&n@s10F96J@F z&QL>>blWMai8w%&p8ay`ZVzNV!DXw9p*kW1M=!^gYTHXeUa#hNg$7}L!}yr0;S5q? zFxC~UOW8-)OIc)fRZN$Kg=9?~#L{W-_E_vSDK{}UFJ^vyLWcnop&jS9*QIm|(sF}@ ztTr64@urH=);hyR_HX_Xh$3qL9J?M7HLUP#mDJZEL+4!XKB1CvOzI&MYJXlMKV$dm z2WSFb#`b^eU-vze{>|133$Z|VTUx2N+=k-E!H()PgthWFg_|bi4T@xnc&jT*JF#W&85Gns!(cK(r*4K)ZG^L51V%pmI3Axq5@BAB_JFjS z>|{>LC7g*rfm7!$BfCtFX`g4Roig2xY54R*n|sP zR-od93&er|*T^yW$1C#@7bu3Zu@2?=X*hlK5RRQnM5!nY&oBNGUG5~3&1ehRNYHyu$vj(^kIZH1<;5El+@ z#rj=mQEPERK#(sSEGm=~7NE3R1Gmr~m@w-ROdQ@D?b@{X%c4a3$xZnD+ugWPqlJ}( zAah4V-+M7}`fN<0&-t(z2}3DsE% zSoiG)9J^EioumzVPkA2ycxnPVddY7R_{xeboZPhqyG~v~d6N!iqXFBg#n=DuwjVECw|i>Z(zcc?HJ~?#Jh*Kz9bK^#lChDLcLURvGb7eo>p8cUL~Z^PHv z5}yT`PcVF4fSST=5Xdq6H0LE##R$hdHL+>w|dG*vfnYWEs^ zwdpttYn9OGOrTf>_rR{0yzq5AH+>*{o$O79u{jreRxQJ~`>&uuZw5gqE3V?(f4qP_ zG65o)3wlj`6YtFF11G5nG^=rS`$u?T$#xW~Md&i>IeflkKB9e{{=8^N8+1*jxO{9k zHtjxzQjG{6-kwm{R4C5QLY2uK-S3}-M`lmPfGz>Ii2+ul68YCpW6SzYIG@)HXP>t4 za}z^ZQH;Xka%iMJ=rv*r9(i~?dWHKz@go4I8BLX?$jiKnQzuU#{%jJ8>MgkM)y?SU zqrue!>+t2;y+|&og2rfpNbZc_m=SpPjn^=#Uwb%+1^Wj>OD(ulIu zBUt+K4LFMdX`2v?pZ^x-4faJ-Z8h@KF5t+4Lx@jIfwuFb*cR@Iyvy6Mvka9;EitG!r?Bca3y{xmKQccB#~mk!qs@=!7#|hfHM0eR(`e; z2d@@GZy^8!%C79f+b>*zOb{S<4aM}gKEkYCevpW4P*i6d&%KWzw{`22D{$&NH48M)aGBq zf$f`d=)w)utMxD%^iVe|p*Bm=uG>(|ng0YP4vmI~LUOY))~l7Mt0_Zf(it2&d=wY1 z=b_DrchNO6T^1Xrk#my(2)pEbZYb`%!AumGdt+ChE? z5N$Q-P??p8otrl!KBWSJvnRaVCHjjJ_uVt`ZBHt8);)eFqMnOh8tp8Wx*? zpnDf$^?UOX8Q=&|htjmO*mvwKu3k?;PDve%6yV}F42!od#h7+&AchtC<|foummwo5 z9tZXx!G#-@Xgl&Pbn7I>#SP2x$+!EFQ>_LZgE8sVrFiGD``{xd^i8F>cyt#w?>u4m zS9CaB*^Jy|yTAP6#$e$KPhwj6ns|m`A8#uaiGxnZJMYG@xA1^mhTZ5b%`OrIu zV%Wr4m@{P*y0&$Owl*I}H>|?;qnA$G12JyGB(!m`2eHkhLE*LI z*tBgw;;-hyr0_$pv9s~w{2A!h_K&m;R;a2=k(ZT%v+*YopKuY`RXRjXUy9z5uBfAcVDl^>9k@rG;3g@7_NR1n6U6IygYLd zJViPbTsea8*KWkgD>u)@b0b%ZU6dW!3M&w>OiH+;G;Bv7R zuD)&I<0_!BtO$iA6)-3Q(Ra)=%$+t4J;Ho`sy9|6nwuI>Rho-SClBGk(bLGPv0?1n z+tE=-$J$SqVBMiBs8L(s)P5jdc={m-#TRh!$axgjX<;KEk~ko^TYvN)I0Tc&k4D#a zZuY!r(V+198SLG+2S?6jLnrq{+41nKbMVoMw{efZ3^c2de>EQ4Hf_iGd>w+jbVab26t(&3xV&Z=v~FE6>4|4C@1g$i zb(Gu@kC;?dxOQR(zFxl%g&H~h+6KTy(4uX8Tadjj|4aIa8LFyWoZS5bHXpnKmDmk!gM1;@ z*Q2#_elr$Ou0;N|DVCkY1 zIJFF@IJ6&#x&7PV#PJiztJOeLSB?_34Y2A^e0c}nem1e?JDVbSbsB)D-uwsdZ7==% zv=|m6Y74Gl&(sf4X`S7UJD;zVPehq^}XRvI=TAaC24YLif(SXdutMN+u7Kj8u z;ol!GfA9i^hWS7O8>%jC#%r&BhBLX1XcIXa?|u9p#`kFZ$3B2e6Kb;0VC6^4arAl# zbQT*xJ@PN?#Nx_CIEVqcZzN_t`xYJ^5O~MltEnUz$2P9S+CAq{TGtG%!E7I->Cg+a zUw8xaALxNLcbFKpnN%oDK8@{Lb|N9Q1X_~`2AvMg$|e}#im1Wk@%a2la9>;qoTY!1 zu_BDkRk(h7FE(yDfb@C+fkAEICNZF@qyUBG^$>f8W5ncHc=(}V2oG#y_nXax#?oup zx@tWRolixb&J0q3#K{M-{fFSehv#8JzfdS7f}Iqv$-=>P-(&xoODL){!mHCgm^ybJ zW>0PrNl54$i*fPzZfw~TkNheX3?>6~Y89H)1~~eJW5kSkcw)wAbn>wa;a0OA%?$tNXFsqo3Sq;4GlUoOnMzuDkU@qARu}uW8YsS%%=~ zNVIivL{Wng+Tt9fmsVqRTp(P2W2(wrUw}O;7h~JyYM4zb6c=j&0J_R6So6={2VPo&NBVf$hXbyk$I(NFaVjwt;C5U+3`sH{avPr2^^pBzj+K@O(xg9Ig0}Z zj^P_Cckoh10n1DCcwyff&L7d0u5t+#UL85*tqshn*V9JQrYhs{b&L7fq4nW zAjH9(^T8MV^z}KscksO&KV>?nO&rJmQ9*Q)iYSQWjC$}LCY7mZ{aHNd>vOpE-Ps&G z^Z~x~*=Al$&trLMA(IY&&u7Q?pod&SsY^SKeg12v*Jx?Ak(P#h?pgCs&Kli^ZQD3f zDi%@d(1sJ<+s3u;JjOviyRn0x8>J$Ff;5m*7Vl=I*+y-33cpx5f`RVt>^*xa6AS97 z)$6I(YnXp-9Vf>4Q{n8-`=9-aH|h-bb+s8;SDM8O@rU^4_%3vi2!zO&54`pz55y-h zAt8YmuII2prDOTEgZ%i_dAzSjN49a2Q7jVZ++i?3KVC}f9rl8%B#B=<_W;9!+H>^$ zB|Mo{#%8saTCIx3$;bHq<0BaC?m!36PMrM8YF;l>(b5uZs+^Ntx@bN}-xEbo2MGm1 zLZ6Of_|e8STsZMQ-aFy}PMtoT58mIOVZJUD3j!T{x^Vs%r`crTzujt&yvrzmX9@rz z_T}uAXIQVd{Y$x{x%d)SJTs0RfQDVC>K9AobB8l=-?I3DT~%H zxm-=FeG#c^EaDF@kD#|gOaOEV9mn-oi>Nl3?Qv}}v8gPXUp_UI9#Z@M4vc-6JF}{( z)f;d6og}RmBde~h=KZ1m>@;K^4`w#fZ2j>UEL0Yq=Le6D;-Jy9`O*3#OfM*9Nqz>8 z{jh`+`n0EmSWMTpaa{PtQC4crBrPmV+`+}q&0xQXKsw1J6p5q^jGDqPcYn*r#|>nU z$WXR%kWhd?$C#NsoL0?WOq-T^Chl0t=Vpv#M4&rm5-~+mM@CI~pS!<#i{l3NWMrr> z9V8+`kTGWF``op23*UcX4hMH@PgjMMVv(3$ord$~QO zsA!}{)6A0V2l&$X?sS(+D3!_Sym}D2P4y z;A_kI!84QDukRp^ojj9sXHDmbd!p&>Bq0PbJv-jVFV7Uw*y3l4v4J_)61jWzI~)}1 zKtN2d4#T*3+c732BrxIJ1*R4@+*~(A&%E(dA@la+hP1ak>Lp>9>yw6cFp$zNu z5I^0S$cjc4wOTEkYw~$$&Fj3kiysvVH%8q1D8D_GLru#8Z01TH{pJHcHfcD!1h`Tx z5-4@_nXlqm z*{q|%V4$wKhL?6M;fSvO3>fk%Q>s7-KhTrZ>U{+BXOLDGo$BLIYxT7a!5*hs>hjYoU zYiw?Du&Jq(CpLY`h0{kcBG8?3shA>>l+llT!?Gqj7O~k(%s;!C?<}0k-r@dqlu0NO z$rv0rlV9yz%}2%zWRJ)Yx+(1W$hFrzp2?~EO{|(N&x_D3*Oj&*RFymssAQqDG@;RnBR? zJUonYiG1t0ngGx<}x|unrS8;e)8%iW{ z`bCf9isL!dnr*aME!3;3d35mv`Z-Gph!`^VEnX|t+{^ z78;vNxa))2?Begv_CucHzT|RhTKrX+d7Li}??eZg3!@);i|2})Z}S(eMpmUC;p^=zPf53vnm>>(P-FImCbc8-A_-sgmPzp-uL9^c6?>EP*Mietx5&anDUx|}_{-n3pviPw{x&w3@W8UCIc(7VeENg6Oi0dVSy?fY7pF`(;%F8X-%cd>k(eFOs6Vu1CTRV43#R5fASB`#pHNX965hslq!Lbvj zbKa9ra_0CU?C9f2K@iic(|ufaYV_wvh^rgA`!t_=2ardT9UkcV;MxA82g z&f}JMCNRiJOhBNMPc+}!pGTEePjjpFwb5eIvH1LFyeA-lJ*U0P#3Ch4R@-l^xt`@W zl6YqOaz50vEhP}>6cEJ)pX_PLndf;WEr;3H4{_=9vpAqjAe|LbiUiOpZZl3%@%=egDYeua4Aj==bKBdK+0NaO zUY&+<$=(cV?*OFC&BffmYyo?82w}fTFLP&NE*n)EYPBksBp>1X^G7kfofj1XD2Qe3 zH2rf{sf?}f(V*n@eV=f`u)&=6%o6TBm%^g5GG<>p$uC}=&d%Nr6ib{Kao>FIxLQG@ z1sMctyQ8zN@!cM+CxgUL<<#cTu%X#l_VNR{C zC1=}c(o{0#!Z{`+oZ*&tA7-ehf)Hd38#0G$_MWz5s)PidzmmbSqC8%^be;)kj`FKF zA7%$vfes-(5YnOO7 z;SBeG^#=Qex>A6cKAng0pPS-u`tEGv6&BTMsI184<_2rp@zKK!aTB+2CLOqN-B~u6|M1OGt1RR73un0c`7sQ1l@Nk6V;*>x zKOD8=y>l0@vbfG36UL@eem}oEB?2gMXv-P@T*dbuAIsi-hj9GVS)B9mWDe~aPItMz zzi-=p3OC=03o)yivVS?J3>(OYo_vqH&ZMxYtdv>TPI1+1GZ^LPM2XCaorXNZ&6i4P zvi&g(u5T#dwhta*T!#?$e&AVdPt0Y#N<*zi#hUyie*OA%M*6$b$vu#R=Pc&Q8#O!$y785b7uldO-t55)by?i- z&Rh-{Fp95!zKs{t3s_#3&%^`Y@~P1=bW=#E@C@Zcudd~dS{-dR6YC1k^Q$+W;P4({ zba#?dEC>{b-p}RVtmbPoN3ut^&h&RvPy~U_{@wV`qm?u?=5ga|4>HI_N&%qAqX+-F zD~a_w)6aZwqe)xN1D`y?Fb^M&TC|A;4H}xQMiyS$$EAxN#_IxMq(XXBK5$<*qLmaag1;WfD1kIuGM}TQ5^-w);w1dW}nG zcBi8NLXfg;uX#LNSVx1&($a4mSd)En3!AxMWDJAD`*6XhyLh9#ff}urnx<-AKlByn4vnCz zqZ9qR4C9;MpJJ8zM_*d0swv{N!~~u>d4MZk98E6~C`cR_J^5W8IDYD8uDf_Wk4nq$ zB{HpSEI!8tz1;14TI$4@X&>@*ViK=jz53ffS1$6np` z7#-e@u5z(G9(z2&lXW`M+|0xcZ*ov4e+EVk=Vym9*`U$UV9>Ft>>8Ipc`tpO92wMe z64zZUZ`tows%ta(!~4&2>Ztx~=jlkXAW-Tb&1XJb%}-x_l;cK?;X_kr^T`E|bJBgW z^mdX`5XsqL#0xxh{sK2HS;(;?Msez#`F!sAXZYy!af}W1qC~J`t(P`mzuE8Kn!}rm z6S-vGXbu`ZiywY}nCbbYEX~j0@r_G4sb8p_TMCHfg5`%c>wV;M7t&Vzw>G$W6->|zP$%q$uwm?sdwuVPO zo=Sh2Kme4u1aRWJ2ib53(%GUb=MPU0VMy2e`DJ`5&9wb>B8jDu$3B_Mjvmeo>@ksB zQ*R}6Oe{aQhNHW9Q4CP-)0wm0SjC-t_VP&lHP*CjJgc#W2bVs=*e<>J(szk$G}u}t z@$$q~yg$r?VgZ!<_2Rp`Q>iwSv|4C18mO#D;d>88*hK@ul^wcv=g8TMxH%z<%|@%8 z&^We+6S{d*0I2Zj%C`^ZQhy7$vuNuXzxo+=>lnl5zK&;|-g0Xf#mZ}2IH8N(Uka~C zzOeBsH6|;qs%qXyxx&-?zvIkaeiQ(TWsdANaweaAb}{#yI?Mg*KjOo~`!KfOgZyM) zDph7Z(|5eWC=VGS^5WFbPO$ctL~YjAa?7&==qwXc5i*=#pD*C8E-~vG_``zUbdm~$ zIEYVtca{ye^xZbAnT^F6yp(j2hrfE6J%U^);Fm=bvnrn(7L8+|lM6eKdWk1<8(Nbb zHVai{Nqm1sFS>|8nM*rPd}}Lx2H11T}p@)?0o-oOsM$xt0aH5NMdfzXU4 z7fa4XSCUweb!JU^1*(WANiL=FwNWMP|G&1>7=VGkcEAxP*I zF^X$0RM18%Yl|{@`C>f3d3zc|90dXry7|O%;!_K_`0IT#l&*>Q=c=n zT=OgFqM3ym$-Ef9hi^@50jd%?ePYM(sRb|an|-Hwa^E^WKkYuo#P;V?EB5kgN*XUG zo#)oio?$m{iCuK~C6UBj&jX96F~rS@?fT8+-W%#$@{f@vCzo?rSR0A~D*U^0-aBi! zYtKF&J)O+D7UJeskpzMRL%R0l=-IDv)2R$L>MXQbO{}|doUcuYrIP@P6>c2z!Zz04 znm}k{MbZvF6y-z#68c9?=FTGRo#RJW&wcMaKz}C*A;=gy>_x_xDEVhgk6Ba5EpJa@ zn16eYe{DT;8}zsMz(#d>JRj@fPpJqLq~4tT@d+KBW(xoSAOJ~3K~&cL$i~==%sKQi zM@D$jC8#Ig*m#*utyKbA42?X$?j^?gIZzN}Y(HQwcc)ahkdzi0^m?jFuW-qteS8%kiu0#L5a{67m9K8SLCep8SNi7H+09=@2u_S0 z{}!(_-Eu899V_!Qc;#F?S3N(RUUD%Z2y_V_$VX<+<3}3~G2z5czW?NS_KNMn$uED! z8;w?KD^Bs*0l}0ABKi$|o{9O*Kl#UIW##E5?CI}9k<^C|zP*iQ+FSQY6*oM6FTK2j zIr+T!W|5abf-S16f%No2W2B!QUj zexd9$dNw~;dziVkTH0u%QCZA`pU!5OyMh7)diR^p_?(7YV%0`{`8mEjtskQX&gS;( z)vX#Bo2ilUOQ*7}tBismW$=)vc{0C|7LsOlJ+o7<@M6MIzCWV}9VH?Pw}~Vc))b^O z>EcPQTr`0}4)%Q1#=i$AJ@qs{T6=(JPVD9fPfud+Zm}Hm!Ww2(Yk$K(W|ky<&#@g` zD2Qb2_UN}PR~m0|Uo$U#^C;UY#1uP+^Tn;HRNcyJTg@!K@B<(2*PcP$r*Qq*f>xxW z)u810RkIi%7u)?GIfDlZ8gGJNt4YNR8(wGM_WtZKz!N9(kKxyc{XqCX)QLkE${ba9S6K+eH#{6Z6xqGx5ZBzC6Aw9Rv_k z2m1GTkWW3mi0clX=J8!$^QnnL867*2FMV^0HCiK!FRtV0P-hAv8Dr+HWo7d%{xUQ( zam^eC%EgqpMDXS9X;hnj)`%CK=X=xp&@ZSbU;p7en{F+3Ry_;OeaXT8asmP!y}R-s zhYIcFw#~}=%w3!qFNsshIHj6YSct2VdKpOVgi9s1_RaI_jG8 zxNgx%dWr4Cvsc7G&UyBq+?kNZ8kM1iyes4ni^kF4SxiCd!j7?nIA-cRF5i~G@)rNt zta|31TgP!-JSY|fx<*XluJkIl3MZ3>m$$sd{$09q_D6eIu5Jb9*pPdKb9(yO@82bq zbH6;pCcTBUnrYA*sIJN7*YkSOK`c-ZNf{V5h_jzx!u^R^tk;-mvstNc$l~f}hto$U zP^@rfhyLR^dCFt_V)rFhsf_m6GHaN9^kWVU^`Jx$)4%U*?oVrKl{aiw9n*Hb$DyHa zbZtL?AMQ$_>St?V*3~k8<&%u`lu{5Y7=GVVJepNctBo`_mNE6(WuDl&jN>BRDFBo@ zy0h1WC;9Aa%XsL_8SeP%9~?ip2V)04%ykKQ)Z5Ig&pym~y?rSHD0d6yqie2Ge}}Hp zW;HP7`zIOMsXJd@f1SEN0P`%xr}*^!9VvxCmyljudbIMEI$>gcc@D27p5>Os)7Z|% z-Z#pFdvfB$*?e>5P9~f=#O1HdWWU&M96tLUCRUnhZc65R4|bzMETVhNB<{Fe)5@R9 zt$2PofbUbu}scV*X%y z$X(fc&d0pg>U{S6+n9TtFN}(yqew&tuLwTz`3cq-Zc|p7x9xRy^>-u$1*0dt#iXiV z=of4nUflRPqdi1~#D%dFK4g~B%KwP{ zUv5X5%-ZuF`v%*EVb}J!II|_Th6S?x@%6aVIAteMi4xG1wH`KTL>{gLPBz0z| zUIRJt@prf{sesJ}D{U4%E3WR~{Chi6A+&HIADp1Z+8Ps;C2ir%0TJvoeF?8r>ROA` z4O#r+$pLhifFij&`^@@`8BM?c#w?mDo>=)5yZXD(x7#GHyHvrSd(hOB-%>B66e&E| zf9}V;R-tLtujK51kG#h3c8w~Nnd*i@CSOeC!BtN)+RvFHfFj?{ z96owFUt9JA|TMCi9o?2lncH|V8b0RxRQI9 z%wSaP1a7)o(Gt%l))Z#)O40@HUbTq5{3HZKzaWy>s4cz3cW3lxM8Ai*@oE)Yx6AcB zwtODL+{A=P&fxx!aPN(lJz>!>Ljd5dv=atb0xb{Y8! zFE-v`{jEmk9(aS@JN4$G4OgkQ{{B7Pkad*LjEtnayFbUiw1FiW>un-X@rCa>F~*NF zk(h3w1G#v23ROSzr_E5$Gi&BDKn#kcPVE2eRyOF&e{v7D7D*t8>EzXpT_U3x74>_5 zBH1}CkWOOZ-y)LOOj=%EF`XU`Ka%PXa%)x<|LooB8_#iJpLUc9ARy!5mv^vCY5ha~ zs>?sd+0kzH>j|B}9of~buE%CJu&yAL7ZXqM^CySVyJfh;KdLXs&UlH-Hyz^HlRNnC z{0G@1I*t=wSj&8!nfijGoHrzlarIuE?GXKCw>>KPzv7;|XFWSZ$>rZ^2 zUZ)1$pZb67op*Rt=av6I_f9YBy;oEbMF@dFNc7%pY{$LFu@h(Gth2i* zjNb}J@5OT@;&E#kDL)8%7RiaXKnazx#|*Z*xpqozxIt)a_W>7@(bH4FT<8j{^dD2 zH_Ut9f>M{sFYj*}ieH>^%dtPmYg4@PGej&|M%xD^1khF%inu! zojhYowtV|HUYF~vt}7%-+_Jg;p#1#fE9JaJ_sB22e?;!F-5meqAAWHAg-f9eVCuqK zS&-@PrN|lV=4jDhI8ZZq$v`0)bJb2hC}|;T+F};vM%_@(GG|LEA8swCu}{K3D2~`z zlV$3Ocuul(}IJh&(w zy~jmcRVB@X9uQpEZB9HRDELIBj;P3Blp@$$>#1t$!sfa9H4#K50dbkk%AL*P#rdR1 z2Tri&ZfbV?nP0uVhfYm2i&w8^Mnd3dItW35PvjJCf9OHxCK_-Jw6NpPzvP_*jd%b- zsX?bR5+0L6e7FGt89{4g+RCr;%#&YWdRzcXQA8J%#Hw4@kev{OLUPd2Tt#(#@8@g) zmYeGBf8gLtaz4{hLBQQK>oP&q_m&8*|%fiVrqiK7>)WQ*p2qi_at$t;ogi!nXAc zP?0FOv7Lphw z0v?=_4Zp-yJa*q|3Ky>CE6+U7uU~zIpMCQlQo{}7i%aw+A#)nD=jAiMa2np*xshxx zl2$*)y{qT5aP>WW_xYdm>MOtGoA)ke+SDv&=H@ZKppdk2X}r9bqwxqIY&%SAKLm!x z5gj#>l1NQ-?jjatC!kjfI0u|)!X~j`?OK-2pGL59Y^p7v)D?W~-@ncm7Gx5jQ+Rns zQ)Y9^9jln+FXC}JXgYa<28ZXmc92rz$NYz$<%R!#j%S|xF+cs`6XZqta9zetc2d3V z_x${g?KtD+aQB^SNDLg=8Y`gj&tS>IEc~=;6rz)Xek<-F?qg@iao+gtpV?PyCt>;m zRxO^2|0OM-3XMP6EAQvN6{K5(f7T>~G|MLZY^U81d!BclK$zQcnOp#4e5IjP^~Q$Jzy1=oD_`Yt>}3lk;cK(%fyv>9k_C zx<>OmYr~kgbTL_B20W5{)Agk27th*wwe`_V@+HJ^y!*0VPNh8#^3g4(m{6zuD)=YV2D~=Hyxejv`@BilK zd~^op-+mWMa^gnO*NJK!@wrQxo#=}~RDfdull=mBw}h!`3%~yP?HL)6(WuS3-$|q*#d|f&E=fLqoszZ(4rorbz4ySlaw``yn=ZY%+11I z1l%rUr^Mu2zQkQC=CgR+1APB~U*dPKzRbVgHk)9z3rlMS)$Mjbz-hORmX}hAZ&cKf zyqdi7a$S^{$KK14_kYbRn<|J|cn5bZ&c<&fm5Hb!G=3KIXD6UlhzQs*nH*ku2_n9+ zd3<3_4t^>TkFA028+X!Vv0ru`JiWD)96f_4u8@^E5nLA|0}9ky9Xj72;!~0dR13(m zKtRM)?tSV>))!5X3q85 zkVPdLtp4J4LTMmr&Qc09!%-_Fx*Kab*Vw|y(WdsM5?=fDTNF3Rq|RQ#(t?c9^SJ<} zE{NIdALi~Q8E7R3bq7D-7k}7Ehf_is5Y3eAnanF#L|$e%0^o7EaLIvec>H0O7Z!5M zL;uQ)zxWNWzWf}w%#24XC<%y}#)`G`iO>o-2fI1E`9n_ky2oHMX=3lLG6>D0U|uS^ z8#RJLP!SZBLiX%D7Ot8{yk6yfc9RL8*|+n=gSSvnxPr(2<464FcfaF*zxNepge!5I z8mMaM#_f@DyPPtxkN(7^h zwIJdfHHG!-XAv?EMV7(&Aip~iPeZm?g1)x{e|7N z{YckgaAc2-)0+7t}O~h9tA}F*(PFcvU>lPAi6mZ)H zs6Sas%LswEObryh{X2G4Dw)4w1(U;cqwJ+RAePzrg~SIaksKDPN;a|cMBCNvN7+Sd z)jodr+qXE|qhQ*qyI3(b{4#n!m46~DAAE?Vnc*m%CXW8~b^frm6!S1oo|?Eh>sdD^ z3B5|d(Ne~)!xi*iI-py2(cVzbkqQ?DE9Q|HVjP}r*K_J)Uiss827VsHM=neCNOPw9t&12AU)U{i^zQ@ROySj|K8hJm_MI;{^fi8 z;?>{r^7H@Bl4Nhp=Mz7dJJ-!2P^Ut+x3GWLVOmYL%P^>c_6Ck0t|YQxC9@KPhxW6I z(1Z+fa`PxCT*Q<(&e}Vd6Jxma%POL?SM#NN7Zauz>8n4)?|$_M zPPFuWdU{dzVC(CozP^sSx~uuCr=htMo9D{PNAkF^4ffOB+CW9wDJp83=&@KYn*}qP zIY`24wIazw_ZLMJpJ zTFzF`Y?X)cKCMoNUrZv&QNd^wAY&ku>GHIAawnvlA}E zmXfB-BQGlwh3v-MbdoI}71QYfM5Pz|T9k=S0H?Wv-Fwes9T6`esEJIOK|x+RdffDN zc(E_{SlHL0lk^#rS+rsn{)&tA|EG4o$?G2+#-d4O_1$a82-aNY=Le?F=aGkRAvsuw zz3VLRy!tzKmbc+nY6wZ3!mOe73oxhw8B@25X)A8wz75Nmx9C>>^*QhQ{pjgiiJxHZ zp)5IQuiVGWKm0Ftn$vjl*=Kod?Mwo-;!U#B|Gb8s_Ry3ZmM)#cj`A&ZI-Rss9Aw*$ zBNQafzX&+E%^cjkm8!u2)~#MelF^%(xIyk5-8EHIHMZlFynu#yiDw`fT2^Bz8hLW$9!AIhm{NtwHn{Ny0-GpGzi zrA#4en99#X?DVZlj`Q0Aq3tQzob5JJu-^wZu=I!@{|l9H=`&U(;!}Y%JoI1&@ndT zyE-_I?CDvAeRzU;Qwv9S@8$fubdp9y{%&(Q+e>Qjn{p5HQ~ZYI=rFZXy!iu4JJc+V zG|<=HM6F4<#Bp=r6A*+}XkyUXN8Ond)R{IA8Z3IjB8?HP#_L%42B%WEd=9}wid`2F z&E$aSYuJbi)CN83p(IgAYCg;I5{3pA6=)0FteOlQL*sjTvd=ycj~28Uhw^7K{k-dmeE)2(3U%6xKCFXvuY>4KR5 zz}NX|UjV0F$=tU-n?QpCaBz0t$80}#7Ms$K_>62OjXCp0Ko^`q{_^FdeR!M`y}g{= z{w_Q2Uc)1~LBr0gN~1;z2udvx>A7SlhKw4{E@%UZjts{~3tcuRmfmjqWeB=v`3WeE z#vuSqKp!`oRSPqRk~;!Eq47)&jUNus)Ztm==VftT)x+euIYb1H4#ot9=nc4JiGFh* zc83cXl7l26NfNTHo6_BXWz&IdZqG}?p!N>l6qEtvELuWSr!V3)YcUy*$Ac`n>1jI7 z#y7W;{gss@23{D_DH4z|izT!5_^1^(S4Rp8f@0Ijoso(EsDbGM8vjY8#TpSy!QJ0Y zyV;6c0uvazW1%WY>NC>(_#)68V;BbNh z^axEBw8mhPQj#%VlHd?D;Y5WRP>5hN_t4gB!Y(niFYKJzwV9o#>Tqd8nKXGC34y9H zWv35Ip|2D&TQ{O%T-gh1B%igdTuW4&aa)!=}N z;1EBA>JA)(-IN_JrR%b zl`m1zi3Iy9s2OlE*waCeT?Vz_O=46ZVbhip`i*mm&Up2%=_~$-Yf5+zPO0}HFq#6B#z;cFp{07b4Mtt)wB44 z8HA593{R=lpwnr_vDq?%E$1lO^fC1VKFo^H(A8Lv<&v%EkwpA`eNhSBIQrVDICF;f zfdc$|1@sX~%q`3(dS?yweXSfW+C|;E3?_|;rFAh`V? z0I5J$zdF~}c9iYMx=5JyATy({*1mY?sw-pDrd_n!AU=5}vyyzrd{>3WkF4eE$$M=x zO%*1Zk9^1n#rKjEJ#}~%)6ES7G$bu}h<~{?8-rSLRqtkM+RrOLd7H|9$M~Oe^ih3e z8^3?;_Y@tgrO)a>5_LqT#w-+6DwPNq(tr;8&XqG$T;7z2aQ z3Jna}EYzJoMosVSgoTKx{lb{BcnKNr9N~o7Oli?JD(}l)29%=@i=v+ zR*vl2!};|YB#$_+p1w+U6raO)@|`S5^&3tU4|LbF{euswHoGv+n9iKEkg*G<1RY89 z*RnAE9V$EP={~cAzw9{0g7gT&4Bn(p?o_7$03ZNKL_t)N!JtPWdjB1iKAVL(nL{&6 zMSQ~&m=boyiU%4?ck;^5U!_bvn}7Y*LljPqL~|1h3;vlT8L3c>F%&LY#*`0=*>%p0 z+g#7CqV1Hgn#r7aBLXtErlV}$a~gHhI@T5@a-+xA4Gi>SwR@}n>^2Mi7KzbQkTfJD zC*Y^kAoNMNU2aAuZl#_oN=}@kx4$3Tq4#*^i9MGMwUlKEr^Q5YL<~WIPGP6V?(!-+ z;X>>p3@avuc=-+#L=mM{i%I}lmT}k}!^tAIy@`^&rS#ftI1m1X|N2HTTJiFs?hb1& z0WonH5it6>F?G2hZ6tOl3}<$IgF-LA$8e@q%cLb=<~IpT=@kM=Oo+mu9CFm`{WPCF z$?>!8I6X4RZd^{!byam;2egoFf)ThXoxjAiPyDTKX$ zjM{z+b!QGy(eNP2sk-6cx@ft?VdF3w0MejVdYv%0+l|ZZ`dpnkTkS~>RWvZ*7Kri- zA~@K8e0c@MF>TspLN^|#e!xQQ>BCet-cM4B4irWvO(|f`wBK`PcN1+Dd)RsWadMW& z4c||8yRh5rxMcy^*~i%vdpUmgP7F;$A78{2L5xv4jzu65#K)EntEtP6N0)w=l40sZUU}RB5?H5GQ zl``o9S8!lt`Yruf9ZqCKY*q^c1D84u8WJZZ;GGLX?8M``w9@0+&!tLh_=NgcruXzg_MNH6ruHQ|IgJR-7y=Vjgk~(|n?EhUC5fmt zYA?9xXs7hpNt(KBi287XBcd*LB3*V50|}|wWF!Vq+S5&6`#FvrJIg(}bI^{E-31LG zDimC<{*-9cYHzYzl5jhnxa3c`mjnT&Zy=$1<@kulusp)+*aT*%0{+%7zlH0*e66J?dP!Kum zP9FA%Xs#8Y5Y)t`q!1Nkq_*}v8(;b^j^0Kdxoa(RrY7R2RiN-m=gx;GA*$6kJ&`Pm zim`RJsKIE|c=dtg#A?3~WJq+JI>w3f?F^b_cK_jr)Na1~?8uS}v)w>+Y#ay*#KB&i z9ybi{egT9lrI?27nBXmnP^tY02-IAkOav7XS!?;ht7$Y3XoyRU!*9rImEBf4&R1}< zv<8dE%Ru0AI*~5)fw;RVDcVO(kCUL(7-FJA$8<^+`fwIJ@CxvX?FlymZAEta9gh*w@0*4{%yaS{7#O(=pgNr{QXI1Xri5#$Rmn&1?_$HO`-$^9PRK43Ri_S8ZA<2B^HPVO z?~@5=g#8i5_Mrr=2F(ztgdiXa;utwyZz3GQ8>IL5JBR%_$<> z4tIo%aDZ{!%@`tMh?7C<52h~XMgLduo{Ni*I`+G(RBBWz3ux%HH z0(T|;!J(+HkR%PCGMDA^ve{R;gPyjt?A%(!hQeer0)|eYY~}QUgEZKKxqbOuf>bvt z5*0^yMh1U^4eIfIK%X=o{QgVKd7vAg=wxEUh7ur>n}Oa|%8!)MVzDCu9*-OMh?a79 zS0#H(N@=&q1SiFim=rkf?+D`9Np#ucr0(E8PIV4q*Oc<=fBy#`=r7f^GLp-V$rwdk zoQ$9g#BS=sH9|vQtv}LNd6YxdZ8&8G{z0Jx`1_2jqcxFf6w~uNxbf) z79Z8$XehKGpi+>;wF{m?4~K*riz&;DKIMCCq4R?@{{ zE~&SggU8R(W9et$=m$Le#C~rA_A)G`i=c>T0%XuC9rW0}_UJ;weHc^G842>yGocz4 zlmx}iX3erJwpQ-J+F8oxqB3sFjv{0z9(50zDBr&uQ_L*pOio0vxW-sdcG2Ea!O_xs ztTsJ+-}o5Cn|f9 z1TDUiF@&qdapmb3Hii7`DE?Am!Zp~+p#!B@Zpp-;Qr-OHC8`KX$>)x{ZzVu`H81F* z<&>Qd|L8=r4(V5n$>U87V z!C@~{Z72abuMqA&Dd`1kWV;<&ngHgbh_b! zG(i*@%MDa2m1xu|?{B#W=^v1Wu_a4yH3v&7ut^S1{`Kd4qvZGFU`w`PY{{q^!j?kR zpfHd$rGUB9-=J(~BW;y?*?GK!nJeOlb(!pTj>5jqo!HB<%DYL-zJO1H$F4vAhU%bIJol|HGc(Tbx(j0enIySRam3DC%<{Z9Id=XKW~YOi zl5K21emhg6r(+cQIlX@;$Ig40_3hPU1!->h6Bvy~v>G*nJcy~gfyTx@wq6N62S)OU2_RQZryuz{y;pU#z3vqBv}O1SU1j>r|_-pkdzxtfal zKHT__Hg7GD-aePWE9gyy#z=gUe%$+CDLXByeMVKwxoy3iJ5kJ@14pTAGt*k#H3ZAZ z+&rQciPrjhn!9>&%L1aPLOGF>E&384pFps`n%aH`y`7CTHgzMV##|Fs(8yHE=ZkAg zqNT2$mTnUsSwIw(SCm&mf>&Pk0}gsR8fa?nK}v~6M8GFDgTnloY&_UVTUQ zMR7!`yuW3xKSj~*lL)^4XdHcX)s#_k zXPa!Gh+yt*_p^RlJlZSlNwsen>A~W-N9KlBTnNGq0DT2~e0hOi)I!@UD9(D6V4DsW#3Z{(v7fPS;;?rL@6`LoLjoY{fs3J`y#AC1nvd z0w8IuI?A5i#gsR6(%w*uRpK%WEDh9hvZ@`MOF^X352JB{6Q@+8SE@&Why%4%RJQlx zQm8Tb1q>BOjN{#@btpfrrN5Fs7PX_~<#7(sTzQP$dy1)O=%B5>mJ6&PBmEWG(@*)y zDoi#P8iNrZqj6mSP7sx7$4Pdo{9`Fxv4*Lej;dWmv(D8J&QggnAewU0Wh^WRa zCb)Q~z=&w#B12J~HsR_wbH26~vy?sN`w3U!4~T|%maSXHpEe!gK)r>Ndy6POy_QMy z;!z18nQPd8WPR{Tx1dmY%)_+WKrG z45B=&D|*h?bFQHYyR1N~4FfWq}}GCt+}4| z&OYRr&rfIhjn9#|>8`5e{GbalAd^*hJxW3PmE@s79}rE1;vZFNBLGQ303?tv`sb3v zMtfBy?Jm%!%;dgD?;tV6aD|(x2#QO^_lhF*U>m6BOjQpaK&@1vo(ObRDDX+hWRky% zvR)f5Q!N$s{kW%tw~j)@7?Q@aHTnFx_)R+cYuNT-36Ct8M4XQrBsX0ZJ1J>3u;9UY z#E#&5vbHo)-`<5oX(TKER-U?R#+9)Yl^>HL{l~E-SGS*3#t?Fstzp`m#T@9caBAC! zoO)ydi(`#|M1NN~yGvThT=X!hF~(~@+a4RHwkpndxKZe0S$f~2EJ_Hz0vITu3yvq+ z=ORcQl9|VX87b_mDxv5684eVmV9oT!L&**|9jErN^LPt}AU`Y}7S5k2;c!(0IdK{2 zM35~tY}<7bF?lVErvzU0d&mwO-PPqZ3`(F7P^#1ucIatzzD&wU#webl&*P%Ks)A;x zn+SvI(>X|@h)Ss(DyX>9%YjNcQJ1@frR5mEf9aQ$N|Sl&2jAu1rPB$~gSokaqPJe+ zhcCX(rr-XQ*yL2cbI&Y%6(h~3h)-NR;VK2~xNtjLsHrjIo+V#_a(3*8?XRn&Y0z7g zfHIia7=JWs@zYXJBa$X#`wD)Mlb8{C;c=aG)t;t!|9(zXG+}Bfr)$u45f{Af!s#oY zY+?9WWn|fn+wB>~maJV>oNcs$B8)k=-p?&F60SgNDxmfWAtU(WS@elXr(oVJ-Y;pO zwY!e}+xPR(;`oc$*ZDGb>@I~1*jHT{#YaxDeCmAs6(C#NIIwp=J%PC_%Za*(eRUlcPCvoCz7ndFFJ$5P#fg}MoA~%p%{9#;x%SJ0rLT`cn>Rb{a@lY=N}!Oz)D zkx#T!=<73M9hR3Jr*mR?g#-uVW4I`pYIzOyc81a9{vhz~OOW z>$iA;a!8l=6Zlx-h=#KFoQ0@HGsxAGohHAku0dVu;q`xyS6g2)-`dFI(Z z#2>uKrqUK{z4h#R^XF6?J;3%`@8`>p-p2I!Ag-qyUg4X}`X`^I&jBy~aW^d%EA_{V z_*rcQ`**G2{>Q$=Z40Mfx$Ek0wEuc~&Db0=qK}S*sS8-WEcEjOaLE!*dmnWt_OfZq zPENPFNy^D*`4^TWHXAs#^(bAJ*k4a?CvAOJ+_H=$U2BJ@OmAl=7ONduRv~#jpD)`f zdvMsi^4h#*C#TyyB+F6J;Gj0w0{-;egp^j zp$4387uJD(Y@f(pD2T{e!MgmnDQ!85rT!S3w;y78b}TVQnU1q%oH*OZ)cfa?BHN>5WIyElKx&SU4%#ZM)F{sz)R8|XQBgsRSgOTOWvtFw*H?g3;}Lj_FNb7W1l zcMal^WR%7*vh$a+G*x%?9}QA z;c(!ZuvJt+SXc;pjRtUGw^^~-Y}d=A^7rA=$sW2oI&pX;WTg+2X3S&dqDlV*K0Zm* z=5o;4-h~?(jW&eog-gg!3c4|Ft1RQT_R!tu?YEO1Hmt6R?7DzT7ez#<3ZV{U&met$ zPUI1#@d~XU83k*Zm$Hd>%Ll0|{*Zljw{q{)AS6c*hqe`CjmT#HoP-g@05TS{8Oy*R zvY^E$GJ{2{*WmMM93+L7*vtZ! zC{UB#ZU#F$Fu7$!y^gpU^I4gfaLtr8f-#=OOXu?1$EBR^Y~k>M{Z!vJkK8B?(qJ1U zJBz4SP3O@^wEX$!?{T*44BPjeF@3tij^en zwh=Qn#(GY*8sEqW`~~QfWL%a$`aGXgltpr5syW1gS~W`_`wI81%pqhbmI;WR%DPAX zjqY=2_|~7!u(N0vcP-6kTKEXriVBP|Q%H}{aMEnYdgWuB7*TC3_Fnle*#&Qi3mbtICpuwYz7C1ppjY8?Fgr4nd(n zA00`sQlts+^jk1H-8UdE6HuyTJcBNr1Kk{Y?`3|Ok-)bfT1bfI<|^fX4sO5fQW2NG zhLzc|-Xw@@qk8vWDJrYu-02dwAL?S}s^w%w`QH@bsUvc@=fS&}lNOF*u#s&W-=gT~ z`61pw3A?3%U3-qvD+Q9j@=k7}SUOticK;3ZxXXo0l0ml8)!9Y= zfam7x9FLvW^1b}*+fVT1_g4l=eu3w?e4SCp4}h-yWlP@qyON61S_#Ef~&pPGn4;ik2^ zl!Heruu3umoi*&*vL9pa3fA252&=Or(TH{`4<4YjwhgD#!nxADoE%JGNnRTMBSBOd zqzhC=l8Kh)zAGeiKEIbbCXf3czLVVKP((`|AHDYuyGxs}xn+UY|2xb1- zJ6V;NfadRJ!+LPJ+;}|TaoRC;cjLI`gFn>9>~dh~>0v;g$g$C&(_i4e7f~uzC_g8% zwDr`nf@yV8!rab1Tt;aqx|UqUf`)a3P=bx zAWC*x&mQ8fpFhi!Pko<_M;fkbc;V$j6$y-=$)n%?A6|I&OB7^AV^D$J)X1UDZ}99> zPx9PvHq&5sdE4gGayiBldR%bL->(%2yw} zo%uN#g!ztLB_TVVI6WR@+zeRE^jn=m}-R z<4^PCL$^_wlTMh=ICOBb6Q|SLIpOHHVD9g~?meOo$zbJ*1tbM)k*pmY-o1q*=S+B< zR_c%J=DZ<|xid2Hza(2JOCCHf2Tq3*hr@CCpVNuc<+^@K(@!E-x7$0oRCZwQ?ZjmM zET5o@T@#Yij@i_Ubs~GA)uPvGymP9=8)Gk&#za=$wt}PpJvh23-tsZW>$-3|%~YN| z#CbW5WwR4M75?_CyrZ}>^-J=Y6d8betTJxdjom(g z-6;VQE|(RX?bIBxG5}V(OvSd~HL3Wm&@GcH?x8eRiBqr+5C)1^SNRjv$Ev zi-~~&>xBNX(V$1EQXn9pP$)0XV*E$FWErQ^OZ_QZdTBNFVgKhe`%f0%%Mwnfx3bvN z-A40(og0cdkD!^BoV|2*J0_s?6;LRB@EMw6grG#N7Drc@3o62svRSks2S0^GUuzj# zi%v2qfwlP{+fSHC$zQ?LQJrNnZnqncM?!Wx>F;Qv)AeZ?Gyz>$GV>NLAl^@by|0yn zTZ=e1AmO$)v3o};nv{IzO});yFbTFTa&_x8BRbbbkbog_FB>Q`z2!+t$I+{d;Kh$!1wjEE>hNNarH$ zk}ONux;yE0PjILe3e-NvVVq10D!*lxnmi9?PXrzK%jxM{0CNy(8j3`#PCU3Ki*ychk<@uH>zvP+1tRmcA3*eEARTI`%F<{Juc@ zJ*!9yg4#nLv+?Kv`FDSvuYTnL(gR2HM`*QL)M^#DZFJOEP}AWgKKcqA6S51d)q_T_ zxr}r3dM?pmK(A0B$Tk{lYG~{1Co^c`q>?OoaM&Fvv{&Hl9P`T_T2Fq&kDvW7{#e|@ zf+v5(x4(7^Nx|MhobsmzoG9@3_s5`D12zWCy|lHp;U;KeE@Bx##OND9aB#rieUe0$ zowU`Q#qJY_zrR0voyIFKQ#Y+`?YIe?P+mjK+ZcTV3A#jHqF*}8mge%tt_o_p>)2O( znERHc<2k;W?bTYAd}k3ENl8d6=J3jnY8uZUW!u3j7Dk41XxBju({5v0Qsn3=c0oX4 zFrX6zkj+$`I)~YsPq1#{nIe+gg~RPZtD(6-K4hFxQ zf*#it!me<{Wal7ONrPUi{2SR08B4*ykU=X6cj|~v<7;u$bYv`C#iA)2Ik&5tma<~D z7nhO~6vTl&6@+Hp%hZ(M%ltN#A&_NXex8I4wsB6ofZ7m5O2ns6YbWS*7?er`9JDvq z(%5Wbc0|a;V3I83uvrnb2GpOTI#Bu;3GnqnDS*>9Ku2>ET~6@TOynne9UxJwCpat& z-%t3wf1{QB9b!m2f(o$f6RpY63q+5K!tu3Gq`Rbb{pS zptjnCd%=YvALA%3jnw_jOZvmm*xf)elg5hu#ohP2RYQz#fhCC zbK>4ybbIztEyeMrrPCR`C4^B}N@slq=h~el#cHmMr3|{!7_^@ZOHukUeeMEgq-~`3 za4R(@cCx?pUV=1x*jFtv?eV1~>96ztq5_rPfEKX0x~Vu>!{EAf^jB!D^Ee$iWf6^9 zHB7yz4bEoe(kZ-myot`Hlk6xt!@6mHoGsbR;l3oES~81CiCPvf$>HrIo9L_B&$i=@ zWVw3TyRVYCdC!m$?sskU;z}hx!9f^BFi8@F&2`ibxJmWZOboFAh$6)8w%r z;uABI2fvzzM}7IQdO;LXYc(jqI}7r%4jTcjUl_TImyo*gG^fojdd?qW)7DcgOf0CWJKtf=)Nr&) zkf`LGbC?DT3Aq|<$>l+0H2$)0n3`xjeiuGq>;FD8-@^(~gBhuk#k zt*I>*{VQDs@aM+`#fIFL$Zr9OT4@ zzvb7P&XGEQDLJW8+|0r%r7novHTQG>Erm?a&1Xi8NX5RbY%VIMO_j<6-~18JKl>$S zCtS*-EaDp&M1WC`C_8AWJ;9-)wM;w@S*G{wF5cRBkZ${pX**E{h7;zaMieBv$`5m( zyq-b#1P>ugm>Z7q(FZ$dypHNF*FX&){NhF4C^~~9Yz7-v6_7BLAo+AJk?_PsVnYH^ z0xnYrXKU){b&W6k{{DV!4yTvXJ!%rk3I2cgPPwhQnBV@-|FY>!KM_eu#D;ihKe&23 zsH&~QG=aP`iRl+*X@Juo|A|e< zT5$i3C?;Q zqf?G$)txJu66JqY@)8sQ#Kr`nQVPh{W{US6rK!g@!Q;s|E$2D3?>L>#%LSteoxyFl z*VS607c5PP|cp+tISS>5WK_I^LK5CSBO!+GzVdvCAhwY*rZ-QE9C_0&^UPh~cV zjGn!Og)>I=CP;oFD#6>^4=+zO;6U3^LuqN_&+ft!<+i89s5wHSf;sDI@Sw$N(APqee2=+o|VmX{w>1wh@D4VB4te?#5!X zgUp?X=n;et`u}2*#6f3eDxdxP6Vgk&2pklGn?eR+C+Vk?sA$z)6Vr)SJ^6|IIh0(9 zMdJT|U1?*LYJx(7Pzzu;*O7SSJgvrSS0dW9^_)4dmyGiEpSW`}Ts@+R9p#)+Av*LF zW#^%_+6J`W4qF%6Zl|P+Yj`XZhIn0FXeqcbV%j`r#YN(Rnc9Lx&Ky0)k)#4V$Hp^n z*p>H0z|+?kUr%>rl9lGF3#6Pc!}Jqeil*!gKkP}S)nxzGcG-L6G?vba!`B5gH3ghK zbCjb;l4$W9$I^MxzqrC#P$=;T3c*h;;INq~NjO4otLZ0577e*4_jCM$lN9eg3n~xB zFS(JiK`!9vAobX3vJ2BWb|8&_8S%^*GXyseKjP*uWx^m8Hd7sE_8%oB^AxEqfh?RA zhrh?qOjN7g2pSPfgog|y2Zr_{@=69)?2zns%o;5Q$;s_SPnf_U)&F9_Jwb-r!yC7L zq}CiZ9VNLXXdEs?j~GcnphrIez9*w*-N^FUQMd?T)iiKo>n2VXwqK_|6w%e^vVH3= z^6PX+f|8I?Q(3WOHiO*$L%fZsBk#y2KK*(>6ynko@ZnFoLNK$Ucr6I6?hFF%Lrd**_y4pjO>E;fn!Zc!rIkE zw_OB5NyON3gnPLCW{C#1R|Gdb_5u&wFbX%xO8J>@`QVeS6t!Fa=eFoShkMacl6sgu zsjUPB_~Z6l*f7a~rt}Oy$#{ZDxR-FNxN-~Y;=|NLkE{9i9{|B9J} zdG)Um>>3`)sIWj>WD=&PBKB_IPIm18M2jS1YRTh=4O_|63aI73jpI;yjb!Y|0F-h8 zM^_O$x9lLhu4@1TEMn>`<-q2RoT;|q{R=cUjpgS!m{f|vF5%`MM3A@VRWbx(-+tYG zwSX@HY@5Bm4E*)ITqDObdweurDgnvdN_OT2YTGOWF|0H+G@qVNbLcKQOw1pH52 zNeR2nip~E2yk@&->m+H{kDM--5fSZ+Yt#hhjEllkB_Nqw$j;2BuKil;)#zL=&p={l z%xAR!)$8RNGMSqe#^9!up)Wtr)(vafnr&fn{3^zUs6bHQH+US27slZxlW5J{$LC*e zrNJwP+0$e2xbl%QIUa#AjEQtd0Q7aqZ2tBLwfcd+r=7OSG`9b^g9?M{XSEXlC%kNi zX3p)}OhVgGUVMKo@4fpk{_*#>`R_mfnKxd0j(e6(W01#hBn$Ub2lC=ja?y_cnu%5k zhuwzN_G`|#lE~39LkXB0)je-NmC~?G}skE zve8^sKu$q51BH*4&JG%zTQLa=f}^K1cWT&g?vO9Z4mf8%IDRFOF4-G6d*mDy9VSqD z5fU7L`;|7^Azso5zT)FVb2TTv-$8nlu2+snkh?HAb^(8Q?H%5FVkHsY3T%2U9c3j0 znrX>y$KKDqt3JP<9Vhb8S;XGIaPb?;g1esP?|*xZRa3%Hh&Hr!`II#Myd+8QueA5b zN&ZI1zW4tH6;UHbFv!;(L9$Srb%O2t5@;D9GM5|{8q)W(en&QXr4rcyB)jxKYV;+{ zx?+wWNTS8$K;|AuSfD3Y#Ui5E@4b|{MKO88P~25ABy%&TcWmWoR&D>1D3XY=C6DjF z+CfIM8A*`g78S#UhyYYFFgNG3W9u$1)OPnvf=Lb=ExAYdYDX%~X7N%Rxp1i+++lb0 z>-$BA6}$bnKz9;mT@l9*B-3IRk$D95vtAO?0P7{l+!!-?JV9=11j$U_u!kO>CVsB$cM9K8AU-hyAaUQk1zzGj2>U zYNdeKUB!W|+es;F!#==XSh{ODyy*)L78~$V|KAOV1odF1&6r4-zZyZZQD2fverY|n zfxc^7D~)X(m}P20M@(nd#6iD_zYtXJ44-uiH_sS?QnJ&UyN^%T|3HoVXr_%Dh3Bsd z3wmTxL(!>t^)c4$h^r34vS!DXb^HFOda*4ojXVU zK(@8-?Yq?QiA?Unm~o@=QYk>}CUf5p>`AS_FaRg&uxQCS^bOx0E5{Lzg>*+3m;@Rw2^Ui zE9dILH7EqXG2BIkng@bN~g;Emls3!Dn?1zZT4&W zV98ZiL3h5~h|R+RYT7`j#IPDzOgm{Y6U``Db^t^++xj z`A-pM0U9nbhcmtV1gE(Cvgya{32(uVs;ctL82q?D4V|el4JE=EM68*q^|;G zYQjcLW8RG6C><6mPk+znn~%WASxg!=-+w$gbkm@)Yt%Ih#0kXq@Bw~Hvq9L zT1>Q6R?{hhYw!f(=Z(Ns_5Z?21EP(_3kUe_XaUO5ajaaqoM8c~eraE?u<_je(1T2i z@BajLE7d}0D6 z%?4yD7Xm}WxU$|ylB8eE^Zj-pP=QBa3^7s88Mub}6gF=;OpVSwAh*=q_TCxT7JA>Vp9A{CJb_NU%rFJ62B-8cbPE!&5&-mz@<1}5P8$gmoY*s5a$ItJ+_zf+Egd~ZVkyut$SPeYc#=f)O?hnJ@Hs~dm5iG-1VtajBRVYT^=6Dl z17?f$sz=IQd>K1;B@4#-BL@V92g4^WWYLTWuIBM2B*}`cFNx=HVE-le&#otH?`M4Q zxT33eShhrVIW#+~HVbz<#9kV$qk z?NwQP`_aGHUmwi!MR9ob`C+0cULGj)1BxZbr9i&#dll*+W-p$_AP*HtX3EZd%j(Z} zl3Uw>rO%>>b_<$@0=BJrmmjhf%)fa){!02iKi4`-mX<~u^%g|H*x5$2M&GAORKjNL zqPnpYyC?x7c87EYXccf#tDJx;OHy7{ztydxsfoPY92z?%29KW0%9S%Om)p4>BQA_gY^9;QExeiLT4-=OyCDu=kWNF~!o>OT2rgGEL zI4;K2O0OU$%wNnHe!&x#pi@KVY!Hx_d$A(^`Fej-)L zN_|x^`K1jA-b0yp>zzy+>~c9V^OJk(T<2aJ(NAl&QmI5PlL2;2CLKoOrG2s6td}Z6 zq|1HmKu|)mwb9V1b^1Dsj^^ezjD0G#C5MHM>UuhRhP62ySCGz(lCYbbsBJJheS)E#n&$4y?w@2e&|cd_chA2$Z1zhsb8;`HEL+N$ z5HDncgSLW0tbTV5M>5LM7_7a&S+tw!sLtZsHLE#ejAr4KAt+=Z_l{xN(rE;HslcHp z|Ik_T8gysxKU7%|}}>Mo(wTkV_H|a!)Tj6mkS4x*Kb#?bKi9BSmu; zwIvypx7vFW7#7TC>FP|^Wp`+xyDMQgHBr}SbgJzc+Nf#jzAWh`TKcS)B;l~xd+(4S zlM^y!84Je*p_GBCHJ>g2{ukdI&7@gl?CoxpL_7M9YEEuh!&eC=CNG|XUw=BLf()%*tBINXSWbKHJ&MhT=?x?()IZ~sn0o-qyaH#U)y;3J@;qEqInGQc14h^ z)SlnL>b2jJUfzPK4+$hWY;-r5aNvvg*^p>v)~a|0_tE#3r0YO6Qr}x7UL!6VFiXPl z`O8^0c^K{rFxKXAJTZeNegFQVy0nnusumP}(JZ+2P9_E^2PBlG>%aqj@r_s5SeD;< zBg1_ZSai)~CRZ?c!ffKAy?=R-s8FC($dL&WcAEvA?otK6=&)h4+WWaH65>FJ(xIWL zxdW3~M6#LaXl+K*x2##Tp{;ME-Rz77MA2~>K6j0XVa}8|0$gNBmUhk`-_4Pf3d~m+ zX~|)wyR`#@==>fwlM!pra|F4P@EMDk7Vd>ikm#;S=F8Qea3r&mZgZamf#|^0-Avlv z&w2mr6#VDh!jzDof1i|IBe?1Ir9}C=BS=Q_P9G<=qM81hqa3;_GIC1kGOCE0u#g+$ zCtO8(IPsR$i@yfGk4}tu(U(cRer*8aOjAqJ{%`qW!w)1BG-7uAxy#=IF-*`dkPFT!5 zX2$vQP} zz=5VNm4nBNF6&_EOW@a>Po}WTj_1(n+_rosgM9{&d6aoFK0cmF5dkRdUF4*vqY0YK z!q~z6$}9@rBbYmT0wLZi1iPO6W8blM!*+6NItc2@eabkSIREnN7d;m=A)cj^B5;-4sLf8JxI37I3uE#6DUx535JhQ#*h)!ocnmXV#NsZOuxM*J zym2+3?@XnsyD!d^u$eSeq#xqb4>!^rGLsqOBCaT$5ODDs%#3A=80Id+sx9a6o@7)b z7O-UY#d(ziuE9}EpF5jy59n?wBBjvAqEHpOzSpCDc4BL={$($#p_ss0niocWXRQ3%*Aot)pXjx}3OUd=684Kx-d z^3?|)k)j_;{DQcv_E+KJ!^CCFi1l|tvNUpP?@2m+r*QMKv3q(WyEx>$PZ&4njTT{RfK{10Buv6lI=b%V+D@k=;LvPS(GwRS7orA*qQ}~b^R=PV|s4l+1;jLe?{=4ItVjg31xOboT zi%b=YT%P<86a)>QMEv5}9Lw8>)^iB6R@_94ucF`IVly<;c5z6g zBw^LI(P8=dSu0Cz5?_7!J!dX-(G|L#EtC=(=!d(j3Z+7ULa9WlP@r=0z{k&@z(9XI z`j)B-o=jeGJB#-xvH4sjW^EO_*Stk%Z4r0gIFDh$p2$QC9W{9z*|VF3N;MC@^$Jsm z`d$Jrl8Di0?zz;WbCBSbL72mVxx2fk{ZztkvGf9WnbMWv3vOrCw3B>#tQ2#19f#Jx zO=o>6_pO}Ih`~N6L<=1aC7jyzBPT0Vta|Akmh`mPUIb`ni}lj?FS#5Zl1U&gSt=l zQ`L~pwyg)56BEXSP>)O1oT80}>?97HNXHy9kq4gqBg;o$K3~ITvtsF81AD(!;G)5R z-eSc8hznCb^f;H3~hE_=KtX>K7Rdo4EOt`df^V*+d44Y?7hLe zrbCOxE&~1EbE{EHYeN+|i3j=o(=Rw%=3wOmlL%BP0D*|Pck{&E8Pu;kNOgTGTYosj zoYA3-8{Bujtkh?p#;X>qXDDi z+Oi@xn-#OUXW*jUh_Roj!%}sc^y>J;kX68&fIf$h0ES$u^2I#Y|g)Mm@r*Y4ejRPiJy-bGP*Hn z>&d@RiZ*s4YJEM~X*n?B2Bt@OAlbAefA=?@`uh%Q3^GP9dy(~PUu4oCkDpEO$Xp{? ze(&8JN-5@OVJn8_Y`*&2OO)r_&Z@;Ti41VVVb)NZdYrw-FAxy_EK8>h$E}BM2;BQ5m@Jwq5)SVryV-@Qw?D>{cTRQ6boC)wtVR>o z9-Po`g}Y=%Z!kK~lqBM?S+P0#dUXL`e?L53T@WNK?e%38mDDj~P&nqsVzP>w37a^N z5dk2Y%~*R9XrfV%p{MKYs{1AP#LriaTrMD4n@Bvcm&22zi60w+Way-(D2;=A5@|Fz z?Uu2liMp0nB3-puM0W;xfW_EE(!K-Sxgd)1!EToUGP#QjDun_!zYxMF#9bEHd3Xij z4!Lwxr*rcQMmTeTohzVT+}KQN*6+6 zW)K&4oi_=<&|d}h7Od^XWaOp2Wl$Ya(=M8X0Kwhe-QC?SxVvoJVdDf3E*p2Z z5ZpDmySuY-cRjrCcW&Lfx6Z#aH8nL-wWOzKtzJFd&%?zGh=UF+4lV3c5d}|}%IDHq z)Zut50n6uY`)u9y%&{n@qYMDfl=KvEk_8=9%xG~heYMBg8QiJL5*k4<_dB~jmy%rb zkPp>n3)tV*)VSq!XzP?}Oc$Ma15>V=R>90mVoo^KO}GeJUV&zyjbCrHl7mjegfas6 zL)3vElT;SpLzp@@6tk?u@`52Y&yDPx@oMtUQ`^#Mxcw6ZURbBYQBg_PqvzS{8%V7P zhSuMHU<1jJ)_i{NFe%QM%)|rtK64^zcVqZ8S;;uRDTM@84wxvOFN=jzMwPr{P7Zxp zB%GPaa^l*U91okg*)LO(Z}p0)_<&O=T3%~#CSmzBAMhn{BUP8`z~9rWGahwc#lJBk zQlWpJ3zB5>s)lx<+u!XSYZk`ClcI@duDE4eeZZ%m_G^ zrGQ?TZT9e9v2izZUy-u3FT8!EBz);{?zd%uD$5zSX`RpVz%cPDkY<&3?6R{Sq0-4dO0xin$iK9 z+DG9SlGgln`B}tXm5s$_b%tAfSGhez2J=fcjprY?WZQ;Vee`N%5dL|3H)>sW~I0jX)>~ic9u;XVTb{IZe99FlF|1)RDe? zE=X0qX{Znha!Y3;wI2k#(VCS0>2EZZ4hIwg{uUCp)89LEzjXD>8d7dLlEIbW=Ut2A zz~*$zXYfTbFB)!y6W`pD<^m*B^4fwDUh&g!dw*cIQV&h^Y~l7+zy!TK62{3(-a@zq zjp$w7I|C;KtF4D*xQ9B)K+UG^=&*I6q1X5^7ku*O8H6%%%r<xbn>Pfp7&%GWx=qt+%(L_3n(<=}2KvQ<~Rc5m0V>2Z!W!v^CIE9FdA&s{9_!m6+12b-;)9rer?sRJW2 zM@JwFxD93^G_C;)Jv`!{Fby(P9?VY5c6}FeTPMNJ^6MO@Z?|^5Cy^~D#dz3+JK=iu zQ`(yUY6RMN^eJ~cxpK_C<|qv5wCS}NF+n;E*z}zEi?~(J{c`&#{RtW1o4at6k@|09 z00t8avn?fe?qypIc}g1xTk~ep1vO}iy2pcz<&>YYve7Xy39}V`yO8B@=YakSmhWEW zD1X=0%(BccDtp->$95+lHl)>ruzvpZVzuodwTOL-nc34IPZ1f@Zb{I}5!6fTNeQ0? zpkXy+lxHgX^qQt}0lsZ?Ro@98@TN)t`TMetLMnt3r4%M$+|PJMmxSm^n`7al&1Qc( zTH}Q=x}tGDX=HnOKq|j$j#apDS0CP`dP{qN3_i!2w=N2io$47CkBEkkN2(XPwdS)H zEYI`qet7pT&W=BhNT$mLOfxf9Mt8{tYVLE+hc{hQdfHO(DX#L#Tr_A;X*Oyytf1enb}zUni%1&ta>cq*5kji}T-b&n zVb;O?84vB7XJaz6#^UD}P=+y@0miwx;~vK{4$^$pME zI_88rPBk~{)79X5)Q{E_nQ4;dhv6x>^SrT0!;wb;nnXwjv;F*O%^C|` zX`|!uX7?njnipYKW3EZ!waK&Ato&Zv9!pk}g(m0oum|pkAo!#|c<1(&rSr$xA9{`t z*QP%`(02j-E`w*I;RrWJkVpp8T$gijvC8Z3Bvf!ZEH>pl&*R>i%kPk1-wlQcDA{od z7>z-md-IDPM#lIX<$_`59jT`Y78Qsn;$-uXA?#LQl*5CCwy=R19cR|a>HF{P>DTzz z>8{@;UPy=sseEw#J&tfjbl-=!GbQO)peEig9VRZ;e|2>s$%f%XOGeUL(j#@f;wn{S z{?d#Er;y`~tMe3M&GAJitczNW$T**vrV$c2ViW=i1S2JOdxt6(jp`emUq}6?;(<}p zLII7%i0z@176%E-dP9FoL;<9LO+fp+JMKpu%G%praQHG4=RFacv76AF*Wy4LyCnn-e>UeM;zb8Gldn(eOvUDqy#F zT&lSm?6QI24DOe5FrTR=Mn{B5!jW38zGqWuK;gf=&fs@BDI6^qBX5fjJtn11hZl%x zORCM6y#-OujE3<ZKCG-jL9a?ZX$Dw@u_klFaXUU>1G`EW)74|gS5lxT>q_0PV+h}C^o#ZW@V zkMjt}liix3Idsf)CDCzC09s?8CphEdP_N40fk$#7d(wkzDGQ5GyT9-7v)uw1NB?QJ z%gkLo|LbuEzwe1SKeTwnxAS+DG5y+K)dd)bQcMRE=B8EHGxS?HoDn>1CDn8-kegRm z^ywbaU#kMs^FW%{r)&~)A9r-Mcx}^MnALB)i#+wiSVBZf%)r?)mTe(-p2?1PR^VjJ z;nzA;4SlkZWt=(J@L#_w3xfB8`7sgG19MI$#Um1aSJot)-jZg<*3B9u^enfp{zyZW z@sD-UVo$^{Q`uN@$!^u99%0C}8G-kCQJY&;5V)6G96Mt8*t>0FovXBOZfV`w$(Q`A zQsC~?y*`>FA{NSGHGz8GY==_-rJvXgVrJh5sjo$ilLVL*k+Ox$wYsLk}tVv)X7&$gah)p66jq?rBz_T)8!rzv-K{lwSBd`~5bN3Ng z>CxOqtR1Db=Qq}RqHXZ_HVzMW$Y!SSli77ap#3Tx>P2YLiS+|L=k-KiuLz*c1EBwE z|LysJ6w&_!$MBTK>`}pgZ#;$2$Vb{xR77;sKuK8{fpS{=ZcQy&so?->Cvc4kg^(f1 z44jhJApPlyD5uG5m+c)=78KxA`3LCJ8bE*{|0^q|`Q*9qae`2;C}m`A^@{|~)1Fp~ z<8F^!KX>@x3(Q!y~hF?R3SV6C&x93Pq>ZU6FOr-g0Lx z9+X%=n%chOXRps)J_-E?8=oaXxmQ%$y0oC<8lI@@35o&~cvM>1O>vGmK<|hc!&Prl zXLZLL`qI^REhCooAW}c07S}Vp_svWhuH_?M7sqqGA_<@ALwX=K)2St%~$K{7dp>3EniOh;d&>HsOudoucY3hT~(&D8XmS)HuWeg;r(vO#Mx1-7v4DvoOkk+cd+XDcmbKMr~Y;+=|73q!6|L$Hn zJ@MR23!kwhK*E_Wt1l^r^^3m;D0y`9L^eAjwd$rje(z&7op2riqdS*Rh{`>?Z+O)u z3vV)O2;6dWDd6<{RqBv64+@=pD4Dfl`I+SO>!B`Y>-ao8vL|5LvKO#~pY8wRsi7EC z_@D*L+D>Y5eVdbAr2NA8GHX+Vy)wjoNue#pxqE&T(_`Fismg47?%V$M+AAM(dtwSU zZ+`A`p8Z?LqgNlPBgQ5rLasSPg-s;R~mXwPkl0aXNP6^VgPS70)+a_(x9CE(G?+*-NIc2minYAW&*u>4Y z88Na=gOF7F%Z`iw&~(PEVo5dpvm#QghQj zER_dmJg`1Z&l3Q#ATJ0ktw-%c^etv|AJ&#vmRwWM%+;AO;L4mAA5#Hcmx zZiUhj<}%3Ua6NZ6dvzjkqiC3wqB*VVx|RDk|JU<5#`pX9RCGJ9aymIHODK z9ewX&=`#eO3>$(QNn#;3D|b%{=A$~^v^RXV2~A#^P_{n9EEKFbu|{+O$~J*j*V5*xEK-)$r6b@g4v8u9bXa6@@E?y55nf#D<{q4 z<|JL3`2J+$hV@R#o-eyv!()Ph4imd(vR_q0p^%*^hRKjDjx(*q?Uiyn43q%VGJ-w@ zS;0=zrQ^6OQM8myY2&9XJZeMdkyJ&m}42;mqwbmf7SQ)HH`1zyEP30hAm18_I_M+*a7wlfH#K%-BE}z7FPvT zms(bEHur_o#tW+PwV*C@!k$&8FH8-=dAdOcx8~>Tlh@Zo4(5fEOj#$STbRrV-iDrV z(NA`Q>uKUUO&0`PvvdWua?0zyQHRUN2$mF#AHRN=6-4db zlD6JX^du%n@Uyy%rm}mWpZlWaC;aX#3ml8W*HJ+Ijj8j8F)m%Z_b^SYDqYBe>zG8^D*caXXN@BDkZH!T?fB|;hB;&lvtPTqTVDSE9S zODIl3XU0M(EvyTxOxI_+X%j-1a>4I#ypXmpfg>OmIloCbf9lIQO4-`Q1if2hiGqazw7S5E1k%O~eh*UoiTGZDlxmO6AaLsb?chsu;qnJS8+t`T?+)(Hntn(`Chru& zkF;gJN5#ORs@DF_;ZLM?xO3~L{7g97JI&DMcBV_B_)4$Mp48o0xmcM~cT`F{Qdq4R z8yq}8rKs#qcvc5Sb3}<6;>X7O_%ivHm)SVaM}5IOt(`V9;BQv5zKCSVq|=I#XTLWm=(l-w{+up%BE(~UZ5~?Dr*a1dkDy8_X!ni9 zcUc%D{93&9e7$PmVqV@f9=$Sg_qiwRw7-1O1+&aeLbi3L)%RvRIj(=TEl9pu$zZB; z_4%G6LLyW6Coi?Wp4638ddZ4GnzG0iXLK+$?-?ltFG(D)0mobWrkg6$8gb{rp>qsk zlv3Xg1hpdr6%F2A+~^)1%$Ub}NYGrAJkEC8%U&M;#8M>{gKz@r9$A-_a0}%eV={0m zhk6UM-(2_vYUX&y;q*c%zxKGHDiENTWEPap)9BmJCV?_j2WN(aMVUZW3CzIxxKxAR z3x~IcXXaSa6O0yXaaSIO%B3_n2&NoJP3X78`s z^3;{N6n{eRuhAE}ZcwQp__6)Lbo?FkxW4Kz6bNTp;{Pg&T3bUAg9t7JA+7RTKr@*p zD}NCIYXpqu z#D6;Rd{s&K7CVRI9el2C?!HIHOfZu9)yR3?-=(ww?f1Tft9%7^k&NkkK^RMoPDktX zWsMGOdrEYJqzAWk}Wjb8w=$iCtgMk?q{OY2EN@|a)M}N{u@E(s0$p|UQ+H)Ltn_@3y zU?gpCUxeg!qI~c?I8Lq^H|x=I71YrNMPIye5jniZzYVfk*1!sM>4(VAsU`sf>i51g zveSj*@t{whFzdm_e(4e8pI%5(T{M@-izAbEulW31RY0iUlw3LCu7i6*%``$GaL4Z% z&1zB{WnR75|4%ObV~Qn;2q$7>PQ;}RGviSHFjavPUzTWqsWk)3@C7Y)jXNnfTaGF1 z)9R`X@vnN%6thZ7{h6yl#{!&F)=`mFZ4zGkEuu!W+LPXpQ@1PsRdV==aCl1A$_nzY z23IIzbl<2#h{K`uK)+;uqx;NEdH1m+W!;!0ntKbC>A_}qV1Q-4KKQKg$^v zRUPra8$lrtLjAsxQmR0AxRV73v)TJzbPk+28ue0h#FdkgVJ7-QW$?ffN**-^E3#QB ze;_uS6>Lqs$PzEDw1PY_%|1UEb<)vn(%t=3DR-VBMyUN6Dz-Acn|M<0%gkEh@PS)S z_4exe-U42HnHn`D-gr_!I;JKWWaw^{H+cw&d|WhRK?tvN#X5;tc9LlpUeBUip|<2Hmq_Mnd^-@``I}V zkR)jCW;l}%c$7^=`KgBGZBpcw6E&Ckmmnw_{}T7V)#`sMlfe2<7Bmi8&SE{t9eq*6Xrl&=2uLvE3{PT|V0| zIf7rOJwBtYaE5v6!(%o4N0467N-vUsma7l3%>GP}vbRUMdM4;=jtXDXq5e1QGaBvF^+H`_$B9mh z0o$!8r;Y;o1X(W~01pT3G~CAMi$ezT+KgCEtteCPN#_$}^hZsXmbB^00i^iGqZ<-2 z4&su%vI8wBOiTJu#PI;YO7L4Orz8Cz%s=wsI)4X>%S!5-Q@t4A2{EG?ZJ={1^aqpm zsY(}qQ_owYC!`Hri$>s%!-vO^4y{6_jRXlR1+?yn9E6Y{vr|5g=mL1qS1pP;EAc3?l!V4C=v@J=uR_BF?*C$ z+&-r=*ls8}Hx><^RF~0B>j{b5{-i3j|Nd@GwvRR8a<2zVGYsZ-c_m);TkCEyAp_a_ zhc0IO|0a8-_1@s04}vdLx%ZddAo~cok;5q*t17yp((;V$NdH0A*o`qq zLmq^dVZG8tGHl!-c?Ajqx|IBUy-#?~$%oC0f(=vH$@Y8J2S)6D`~$BJE7S(Ojq`5} zAy0Ea+^Gf)<)3T6q)wf)pWp0)>$5HHplu${f3hm@q7=!7j1`wuw*T6xmDKBz+5wcC zErud#-|TZHjPho3>M}@e>Pg6$LvTJYOCli%|Gg*mI`s&JZp(9UC@xry3jWn=(QVe4 zkBHwL@se|JCnPYO?Aj{k?->|)zA3FWInsA(ngu|%pl()A%m0q-(Vv-~nF(M1Iw6$ecol0g+@W;! zobx`^Q)8@25pNEQ1tlT5ZEjOiTMW~otR$z8vCm=GjAZz4wyJ$de>Uhpd1v7anf;$Q za=l1v-8*S$zi+3VTxUysUZ1$|6=}Msg3`prB4-J~d$W{q##*WV4$8mujz+ff1Wwd9 z7XTe*eGDBr;2h5=^;uGu1+#eNm6&;JVbh;1PXpd9zT4bBbFhh(g7A?U|AxBH4_7jd zrnH6Qcx*__8O7ywdeExcyXzh{{vjKtlqZ%;3b zr4eym?GKEf@+OP4DF_#7kmBl;QPX@vU`jjo5#vYG_}pD^__UlcheWUcfW=^B#rx}) zgr1@?W&B_)9v2-O5DCBXs4560H*hKl!*&Wo1XE%6osfvBg#2nF_wTvX#`@)ck;* zv&IR%hoT*&vphwI!NCcMRS_~mN&=xb6|A7=~lRt;#+ACpZbV_^7lGp8VfN{I~#lcKG7mu|CE-17g z84u%kp-n-{-^hstURcJ$_;mMxa(g69fk$$nq9(|x7{ANOrR64%%HuSTPn#3SDi+l7 zb7;TW1JPvcQ46XqvYw+UpN&tq&=wQTB18CJbsGQUX^zK!ERdCP4ZKK!56M$0nAfM6 z-gn5X*D1m}oXZk$JFnZg(n}5JBNXINAQp`oNU-Yg5AQtBm+&>O@mc6`;4+ zVacB}mXWS>j`4>E0|!OR-rn9%wb=l`BefnbXz6y|tMmN;8j*aDuO>tXvQd%t`ex|U z%U>$1H1%8hRS#XDfD0dcRaRP-W`Nj*!em62u+yJLSx^XIpB0iG6;$#5BJ_CuvYt6s zF~F$FV+vn{pV64T7R9cj${5Rniu$hP4J0I7R44Lz^-f8fj$+5PVlU`l>99;;SPcgu zqoU5z$D!sTe?vErHk8#)Kqc%^)mdqRL%8HucoE$|{$WNEAo}eyKpWRm)NzN~J+rT| zVqS-SG=X8f9LGvye2wqn&2 (tJ6YfJ!i@YyYYPoc+Srz}0G)`m@p z6fa3Jn67lTg7&uBwff|b{vvOX$Tf0r`I&kQEm#8$< zjqyAPEb8@ry)Qw!-Xe>=?yIYE^q>tIwiV$}i9&L#t1=}b=;$5lzf;2+-^)k{oHzI_ zaWk{-66Z)i@@RwvK^UY!^v0j_a;V;HOG`}FFX?gCfY`1K0a!Qp*#R{OY4e3^30M9 z_!_IEod#78i+|I1s&>C&@Y%=P+;Smm4^(M`^-7rbopicb{}>4B@(U0&gzq;S$hUr! zf;w7R6xo#iwN~pj0w-UBpQ=!dpL$rVWV6HczGKRsyhrboorYZH@B}I3w?*{|R{c=8 zcw~=Q4wRYLB04W>`xANP`t?uwNOhXIncUX~guio1tKpCWk5p*HGKXvqd8v^a&buWS zdl%==-~3yxu9`B)K^Wc^2{GAUQr7P=1U)@D4tPw%%o;&CxW*62dLNoVM=||v?S%$4 z$Hc#df9)9N86h1u4cyyXvk3UyFF!FQF2_Jhm#2+>nJMcdi|5a*L_V4V_XXp2h1e%v z;W_W0nC!1hyRUwW#5BX~b;z;=AMVVnHfcqezkrjYYo5yi@^ zbUFK-WE;F{#-s5y_-SFT$&UkdejFTK0~EW(&5^sm{iz4=q-m~EohkkUz^)gum4*hN zJKc{ys!A)~j<8H}7}d(i=eraqs}t)AogXE08W-o`gw7QuqzHs;$n^LDZ!l8?I}p@Vm5!tJZg_P(jY&|#DALTeAGm&0S*k`7bs zmcmTbjDb!Dp@Y?jwLy@-EmbNrbJ}=mnM^2j>*3*{v00Ci#JViL6huj8P+LAeG)X98 z3`&+FTGplG5>`RQ*_+Sy1m9zy!bK0^>Hc`OSYLt+r6`PbPi1@z^b&8+ z)AyIh;mPcsJBa1iu|n+N0mLaG=6}I@V5it!W>AwZ0%@JawJC> zb|a8{W4c}jG+W$UW;3Dvz?HsfHR|gV-h&~okK~m4K!Ac*SuTF_cZsySY%ne z6i1k|H4Y%$`j6ODbstdV1D9Sr4`qOo;)x-Dpu$lPXksj6_kh?ssHplK7 zjrzX9Bb}FA{QDtegQGrz-Q|4_JoV^6a}e#Z)sTjgI$CLpz4z_-PLtbMoT;aRf3(q! z2SK9dO{4j~40w7xNY>8zs(XvcGEwUsot@dUdxs*I?W=6rZ(HCY0qL%qUJd^7E`59& z`A^0o)7%_=nZc~Mz#}#aO@Zn4PwjBTA6Lkby7B=VdC0&*`jF<+10JQ0zRQ?sl&lGQ z=S)0=-X=`{D1c&0eNB0nJ+{-vXb zE4u&8FyQdih1;=Dy=LT0TKl zuXm-fp`wzOV$(u$1`YpwV_2@roz;rge7$&7OViM;U8^Jy4K8Y`<^_Ql%C~faR)@mO(8amd$I!mWmM>VpF8f7BnFDJn&F+L4@q5qaIQJ znssT3rTbBx#WK?Dd}EPL7wf>x;I_4hN`gp)$lL+Hd7|bLSxQTn&UFarav&l*d~meP z4a@FE>4B-{lPmpeW1=t)yCxCidFFOqg6`AaSpab|!l`4e2dAjUR%6^~Pf+MHi!p}< zW~8y2Rb>gxCc{6xDE9s!>6wi))RQokL$fI#rE%WNa7>nw?n=Ivvb8A?veQOU+>`|HQEuXp#KHhYcI4$ z0{qJl#R0(a$5;%4hWA(dCNz_;?Ex~ySlop3$Eh?>m1ApgEHUuz*9X~(h+%p8#snui zc%_Vu1=54%t)L(9S0j;3GXn9W#Wq|nR=&$KD=yQJo!|5RVMBZI9vu^rB>u`D;azCt zR8iHIG(O+KbNeJ9V#n`e^ZJ#2E-3kiGE}3g(-2zRsxu-R>9m&)U^2qRjnB;y=4LIi zfF6p7;OP1R%*&FW3!`fOnW<58&_)delTFsgVj{6FwS=u_K?K+HFVx!=HW*`=BPK~t zNfD##=OVWR=ciX6@Hz3_#4_OG+g+F~i5UX<&eoxS$4!^hAb|>eXXF;8Bbv@fB60(qMcf9air?gfbAS z%rY)uV*T_8*GPpn-$@_uF9My@Duc9HDz?CHhn0WUD$_MK3`2*lM+U7HLA2F zB%HbvOovXJYDlAQ&vwTJvOE%a)VM0ldsn<<2{@zrzMF0^&#tTqGMLsbQPa|pDJi1f zdZ$H=$1W#W7ynZaTs0QBPqgJlA=$z)_6JbC_3nz#&`KoPSXz~pw6;f)hPf?3Wo1X8 z-M)nPm4G+>6Qa^Vk*n&z6Mt-Ha6fTO4$~FZF9mqv#(w<-=>2_xyiRcDdPhWJ%2eQV z=9R9Iq-xoekB!^C?c3gd#(_9!SCN*Mh>_L2Y*BZU5pGkTC=cn%rAkkLaOuUo+F8hGr8V-l0TLg8ji73Cao<*RYYF-$2JN zkX@B=>wUv42Xm;gp~!42C$1);L)E`-cY}?_NHQ5bw-YAtOtLyRKmgw;PEmdvkEp21 zh3F-$gS8txl-F^-zmvfNirYvdfpWw~PPw!pB4GiM7KboAeY3Ii^qA zs_w>2M!9AQAH5BZ`AE4v8)m2MHq$-v=QyVA6QeAw8fB0ljEk1E;KN zJOE+*Nt>AYO7GoqHi4?=2@85>p(LHXiBV@st68~r!r%i4%v&GQkmh~aEr32eKi(Y@ zhhILCa5odPemGL%9e zcVY<-ZZc%KM@iUUk^-jR+Mgu(#ImU&IT&p9R^{!A&;(Hbfpg5f48~b$_!jngsA-)| zIL4pk0p$OFUI^a1-=k%6sxCKYyCF1GP)|$%ihIL6-}e8W3UG(JxZ$N3*2W|i)oqpz zTgr%T^uph|PqQ)0QqN<`r5fSFM z6+{N@lCX2cMyM;Wb5-W&&t+wK({H-;7VfG(cI@JxUK9flf3PU@E2|BRJX30DrtHvW zvS~ng(FRVUk!SPfK`m$EPbHZ4A2AN(D{LbL|E5yC8Hls>v+6Nv*rjFQHE>)WVkRIo z9wd4f4%-}S{ln!V;nHvA$2wxf%NsC=NAB4uffmoK4X3Sql&y;HMQIQdJ+ryq{T@VU z(i-vSQx4xg0pMGlOTgX&d^6C1Pn^)U+`$jFNW;rJ<3$S$fu#?w)=Wyo+ z%8aLZ{pUF3>BbX-_8g3fG}EvvVQ>VaGVRW>Xj@*wEG5t;h)i7i7->65fEX7%LTzrv=2Aa2ivemSs)^lukC&*L6zsH+~h6J{{c zmxatILp`+jkhG-zVJg#^S^XB8ZNhqJ+}xE>`6lMkD>DXJ#LnKNqNytr-gX&GI!#IP&oJQi#SQBB64Fji-Jb~HjgqT`ISMYp4 zoWC^_&Z+r{JCedhN5pZv`SVIwyfH840GFVwzY>9k*vEeSngDf2B%>W2az@$epcqKbO^T$2U-?HsvO zXa{-Jsc)L0o=lnP{YpVH{Pay&&P~?ETcFXh+nVR1;nhYmM&!X@iGg<}B|bKw6u`w~ z=bOgWhRA#lhrBqB-YQ$FGuzX~)s-^3xS*_jke$Vsv1ISn#A)3DNXvb=i@KQ4!wHY&N9CYy<`0`5dn<0Klk>eC3HG#{_1B zqgBMJ#~cB_wb!^1C8xX9 zFSIJv-n;!yF~H3|O&GPJ9@cgKkf$oX^p0VkTa?@EX30;F=+z%ce}=NA)7)}j0oXNT z<76!bLpxXooN$5sm#$$QTVR<-m)iwS`9zu1dGjboubf6QK|vi;3VGV5CZt1VuuTzwG8XE zX9pCN7jLxvP-*c#Ux~}AesW1i?WccZ5A?1Jp+8DYGh)>=x@M|3@qWF~mxB!r6ddmI z0|O3I{A2vvZ+g{M6HlAC0TCV(F+wboEB=!o9WUO&8#e@mPtQ4H{<;8d#4+JTj$8O; zzuX}>L`NKf=;BXNK8Vc1hLwrwn6b zC)68l>R4CCM~{xsyv&9B(yeYt-V5?<3v##;h<(nhhV>nahc+bFy>Sz{hPd*C zYu!x_AE*venUD<9@hK7BaSgc}n=8}I6k zSU#arogbdZ_y_yZ#$KG0D(L&CgMwrr0@dhKi`6v=da=b?&-V6z{6-r~f;W6r0~%y( zXuBz_2ARlJ6SI$gBnR12gWmmxj8@8b3?2Qh2xq}8cl9oyC ztv(M=H9ktB8c8>f0x_GrP|K-BrK8?RncSR!-AC;`7aWA~IHk6Z&DMrQ_*M(zKO;Pm zbc4?EY!hqm4#DciS{0%=)pyWm^D5^M*A@0IZ`os{x z+L<%jQ1x^|dOgIZ!1$iv6JEu$XMpQHu5rFP$>-!++ zH?>FY#s$jyKumuRS5I5c8}MsXf`bwuAWwzNBIMR&|X{hfhf|(; z`zXtQUn?93wDzsuq$7fwZGs+k4HHW7O^D(3{qZqq=(e0wh`ENy&z(BO+u%E*+tDvwcF# z(#P8!#LhDFHv@@(1QNIR!{E=N5N#l9?PI^S8RmVE`&-*g?9S%M)+*Mg6KAs2$T$ZK z6()Q~LSkddB4K+(jeS4%cl3Q$($YR9Uo_!0?(Gv+QbNmu-xin~cTwVxY1+{dZ0v-s zOI?Evbfmlj@QQ3*l8DaY6ux75-Y#mYUS8RhI$;=PsQ=s$R-y`*552@cHTy&a1z3;2 zwEVT}$H8z!`!0e!0gYg|c=P(W{b9mv3{a56v$3ggwe|dM=#J-eh(~1c221zZ#R4g> zKwqZotyq7Bhr*=!@e@n^-Q16-oSQ=%TrU3`%5>Um2_odr;)3DAZB6<^m2NKKE=-@% z{b{vNE2y2qInjdah&g2#kt60OMjA&gX_qD1z+bvr!4B9SZ&3C=7fI}ki6%|jaR||B ztcs34zwh;{rdjZdj@O$UUrHQI)Rki_#XlQM|GvkowcZ%+wMfYO=KK1}XIKCnj9zPg z(r4F5i34zu;o;7PV|Uj;oG#zt>XX4}*u9fCeLNwk@r~u9K#CwUs92J%^&%dgh2Gfo6~G1&=O%;46KbxHhTaG?QDc)_hRLgVHMFz*Cr)rTvS}Z|LKycw#Xx)`lHA5KOI9q@JV-gm3!a719yhy>@>XXp86aZyW2m}P(~){ zeTJl<*I1%9R7c2Fb;cmQ?x#E*zEl;GZj!GkAqTT5bg)T&vopq7l#q7QE=O&;BFAs~95`RlXyJW5>``AGJkwm^HLL58@a!mw9x^!O8uUfk3_ThcMcvTe6{_`V}KF@}Rok2mTNdk5yQ} zT{-QtRab!~pI0)yS&eA*F{)23`;VeUW-gJpFauaEhd;?bxy#hj>$5*cWi=3vu03HR zK3fDmWqlxVX2v5(ap1T3u6)Ui<={>%q^`l|L$nC3pP#z#kr8qaKYU1@SBA|C%jx|! zrX4?C$-vB)I4PABJtlz@y}}a_7N4^(W#NCS{4Qdf@V~|NKSKMjD#W8QX~O-#*Z-Af zprbNs#(y>Ce=h&`Q7^HNGW`E@=f9$-sF5N4|Cie%A@hH#?Ek*lqYnr;lJYCL`a9s4 P&rL=`QM^*rDDZy)&=rH~ literal 0 HcmV?d00001 diff --git a/examples/huggingface/pytorch/text-generation/h2o/imgs/1.png b/examples/huggingface/pytorch/text-generation/h2o/imgs/1.png new file mode 100644 index 0000000000000000000000000000000000000000..3cd5c8ff156d77bee799db6ea56dce80e09f635d GIT binary patch literal 360013 zcmaHS18`p?Jxy7ad;SP7!VK;cu5HnB@hruNe~b)NodILH>LFJpx?z$I|+415D*ln|2#jo zObFaSKnOu3MFdsc)-JMKom3W={`tCkj7}6jL;gTPLGk$6g+VokhOGFYp7VC0oztZ! zn2m>CD5)6xiyIl5j_6?qB|cV6=uKwrp>3N$uuy%gq!jVmmM$eC>0lzu@oCdEr|WrD z6G>E2MCiZUo^7y9I_S^;YGFMjO0oZshI_VQXrytm|J7bJi6JCO|GRx9O9-O;*U^54 z^^pDlh7VU~C;9jjumWCsph4sEy~DdR!I`&>Yl z-3=4E@I6uA0u}0M;DyDG6SqBgmWryXvVwwuw>JRf>+4HITzr3jKTxSG@t67@b0J~a z-~VF+1bMpx)~4$GErY19HZ}| zIt5~zCVv23Kh&UdOVnEwAg%h@Hh1D=C-z`X7RYj$HxmvN{+tfv~Aj0 zvE%6I=*Xlp`rGdhAtNIbK}eI*3&F%oNJxC7m_j4s8%ap2s5*uN2ooej7=B7Fhk`Oo zu1d5YF|e5%II?d59k+OGI7SXs-4W64I{?tkSj>VNn?2b0;sCA=)w@yghE??)VoqW> zf-HdqV>(zq*ZuFBP-{d11src|s>olRPt%_~@QX!8R+}PkI%U>+L-Bb; zp*ZdiLvm`rQn9vYph4katenr)dDo-)8sRTBvAMJ@4EMaJQBZAKA^+z2e^dert))xU zbj53w5t$mBPHXI?#U2zi-TcsXeWkmrNl+=sb1TR7_s6Pmx!Y3~s@^Fr?FeC{KsCtc zSlVE3Z^Ir4xWk`z*MMD3c08qHoBhkWlgI0enj_Se6pfllvNj-C9@i4wWRQ%mr?kU0H35keqE$tT$>WweO9ICz^@c9Dd;)eKlB9C&58$~oAX^$QEj6=`*ZavX2rKD{MFIj^Vm$}WIse5zO zmH8;PJ$ik80ZgUFp;zS8Qgm)Xka;OU_v^%p?!?H2MsmJStf;o= zPf=59b0?S+@4(PVxZd19gss?A6;=ZW`w?=Ebg6Rm)bdwE#*X9$YO2AA=aQv5Unw3! zF-_@)@6U#?XEG(2{PbwzvzPz06onMc$=F*^0)3H?^?NZF9C)9|EiC9Ly#tuae09@q zInOyW+tV*sWl7;M1#?`G&5U}Knx~E$_(=jgvFSz)t0{Vc)J9cd_~{nmT%E3n@30VW zQFwx0vB75n37OO>$lF8+woyVb;h${*vVuddogWi4Msr#3Io_ceAuhxU3?+4Utnc&e zwQg6cULTjp2TUA})AF4?zeS7OK0t4lEqz1o#Ocdnq;FqzxwHM4TG9K-H|1pA(D8;J z|H*qM)zgV0E)FlMs~+6ao33D^Q=ZEn|M zq1xhF@%z(w7lp#QQ3De`s(0X=6@R-y$_pTUaZeH`UrC{rF-Vzy4L5LBUi9y~Mip7- z{_+Ert~%bK|4<(n%Sh6F{7A?s}Dt)3PN z)pIjK4_uWTzLBxPh?Pc;jB-c{<}^QX;) zX&nyK>~yh!ujx5od}_zJ*Mr9r^|&e{J6RCP>8L(&=cM`+Q4HszRjXGUWSVCD{0S@T zY>YJQ?VBa1tJwquB7`U07Urws42ekQuWOz(?WBvBCsZ9DQ}n4n{juWY_L>;;y25hb zU6{(%>KnrZg(t&>D%gJ&cc0`Jzd)ZS)0$jPk*Kh?BZCO$kujFTXC-*fFSUg2uQoko zy~(!+e`M2;8B|RfO;t?aANq5@ZPXASwUsqK!~r*?#kE)Mn=9XnN%k4Z!fm`}=+atq8C9g>Skh=#8d>7cDWZzJTO}R9KO|Sm zVbpt0VA-=2m=6P?S`B45WRBmw+#rT#Y(KOIhD7tckKF&DX}ibqK0;LLc0$gNgu%&& z3OwR0&nfHvdn2=Cj7lH5d5IWS#ty)8SZ8rc+v(e}7OH6Feh(N{pDKRMxu5)keB1Oz zeBPMJ(X+lcHSg>{VfTVzsxoA}#UwD%mhP({r8TFF>0E;kne)q0^j!VgtDZK!%x`*$ z>Wa&=*Y9!f@{Ou)SkBhF4PjApXAUhfkt3zQ**jKxA4Va#jPfe+o8u3EvNpW=)J2E| zfp%ytM$*7!YqZ9SypocX3W(ov5pMO=;FItEf&enT-Clc!AmzqSMF2^H8JpLL=aiD` ztu}4sR2TN+Mv;tQBK;K`uj6H6T()r6lBI{um9bW+PvjkwSI?x>-&Z4e8z?hs6`dQu zZiv=GwCUcJ2Xs`Y{OQKk8i|uG=D2wSKt^WXgJzoz5j|W&bD7@>>;F=1uyelo_He`v zi_ONdb-o@r;@XKT8HFA;;jHB;OAXBiojiIxcmTGxwvOK;M-Cn06B4i(^*}T(55F+@ z`T0LSKFqAF3d_oVf)vb-I4^694@cwq0!gCA2d_3e+#eTJSDP#`0wLk|!YHzHRcmyP z-wU-uWyneR__8Uo-2wsvK)>DT5YKg|aK)dhYN=O-Fd*O?Ck`~x?G)R!KjFwZsUSJ( zc*eit$nD1AgkJMR`e?7ApkG9UA61@R^slefcBXQdWX7)GM`k$Sa73R7wzfDWy={zw zWX-0;&L9Oi`@g(mwpt7m#UwO{b1>~EEvu*06anFtUyozy!Ob@ljjs*&*U*k+btR3R z;orLD9flVy$qE$21tg&JhK3UFO zpn&1+A#aj)}j zX1C7wZ>N$5X$#a73IMYQ2j4Ml4d~z6&+V2JsZss(C(GbfIWB6ri3rO4iTUqRxtfIj zAfy}2#=@cbuGiA`Zc?8IFuXO^P~d@-k6Om-{db|MLY15L`fBL zi_kOBQh<&F852cDAvRmj7qwg$w?B|jbr0iLR7_`FeLXfDpQ`!U#t!;MJY!(g9!rv% z@bn`r(t%qD8KYHzpY;?ipIb|$`SZ5u^^y-|A>BdD)0kcvEg{P%bX0kCWAslkPxD5p zV&c|I!s5-&kV>wm6s`f0n5;f;P4y;wj6kT!!xjzx{9(D??yAbQ)(j%BVV&70Ws4|q zokL!|WPcBkGfW=SPC*g{A z4}hK4<6tFl4}L4H1@`xcLNS0nJ6dt8m6i;5r8#@?od39Ex}xr!xb=Mzry98>xbqcK z(G7{L`P&<_jWF<7hH_fF@WPN$2YuLj!?x7b#r20Ea=b62!F=yi-{zPQ%J9VGlYJ)sNLc6`B_CB6Vm1Re7E5^9haRgw_L65{&rd-DJR!miZNz<)Tt$>q=eGt@wj&~ zTY$^wl}e}C{7qrQOt4al1@<(=ikDE4@`twz3w^KMj;uzCB{6zWpmcY_daGurji3WU zPj2;UYIgL_S|+|6@GV>D4wJwkOMCrK$F&Z^ot$bFm|D+Af{t8XGzL3vN!EOVw z-d~?Bmk>qw8b~&DK z?KAj7jYYRMl-HrLI{qZ(LYqD+PE&vC@NC(6GsCIzb{Y#yI7Bu>)yChY>fI+@<2}Zd zfpe>Y44uZ|ZR;Jbjz_WgU@j@*1mmwBRTsDQ3nYPND~*K(HSpqTf5elx9%OOfopfL57|2n( z#zwH#kXP;qd$4@n-kL+5d2c!P$I@jk9S~E?X#VT=8K4CN1Lt3L!bXfHKJ{F=mK?NC zJ(*2jO9ps@)Ni~tyv^UtY(LKi4N512k{-X>#FtN+Gr z@%Z_|wPMu9FRDISnUA<@tSMMIwj%s*Tsj^;*pjc_@tK%`Pc8JV9dW2UbEOE8MZ;wKWnKN4?=Rv5k@UkQqPTRQ)qi@x zp-P%3jE5@G&NSr>j?Nioc-BIt=)st>3+cN~w{6L;Ki+>QXSrzexYXv7m*2Z>WzGjP zw=mSz(JAP**R|qsW+#hToAGX4=+4G0O&$V2x=Eos?{ZTM05rTlnm?rbx zq}g_Rs;XrxT&E@l?Pwd9N`mc1UZnuDMIh*1`$Ep6>00q*= zUc-b&%3upYSAcT${^sy7vau5IGD=slnv6ORM^SY#{F|y|j;jd@9zPM__El)v z*xJBe(tFyBEl8M8N;iyDfQB<^>EiF->}nN=b#EQ4Z(_Xu zE_#O@j>9Xx!6lNFhq?#cg&gSp7(oQVVgXB-5@2G;i|Upn-&C@;eHWeky4r&?jHG;Y z(KfI|tuS|hq8<-G0<4Ha4N{6oNx@GNYPAqV0mp(VLa7>kK$Vll+S<)Z^%yo|#(#}W zyU%xjjywN(A0Ke(o68g~cP(!hi*faVqQvJJ@~d0wi>a9bxi|tWt~?j~?WuX^Ss0OY zR#1J3{mAzNq2Z5_QiUWjg(Js04$O=j1QNz_us{Kf2t{(kEAEPG?8crHHXeQ=H>y4z z>d6{g7?Po5-_QKOxymZqmlti5stxg~VnE5$rO7(kei%JYk|@QtGjQMa#bENs#9H*^x%H@c(PI8Dz{(}NkJTh`u7c-0HI+Y7Aj}VB z&)<9@a>+9%c7cSsOrkSnbr95t@;uEVn&?;=$}2uEa6eGr=GfF+ou~QoGlIv25H7dn zU3%5e@N(}Fh{^Lvv{5wmRDdGQ|KUzInC}SqWI!1<7sW7n30Q8YQz@^;Z88J|#7E;c z+!f!@jXj4mYn$)&%A;GKD|7VB_W}|u7aGBM8NSEje&Xj?>+~1jl7!){B4nXCe~lZk zrPO&&N=37Q->&2uX#~CA2=hn_#^%kv$FUy#!lz@%eJSXqSwoUVFHr+ta<~O=%BKuq z`m96n#1-G+E-}1A+P{D~Np^ZoFF`dHiVKWa@GwQAG*^CRC$P;>xMKn|J%DaBQb-D= z4+Bm)|43rc6Yj@`EXQl|Ziu5fSmPUvzhOXM;f@5QY;=6~)_Zw&(o;c9RXzP*bOJIg)F(*ofvSFuGi*iN)YY zNj;vUc7TF{&YV9kEG>Nl6=542`pfl}|9y6~2q1xfN3B&c-(UhzA@8Os2#N|%ekS8m z8r**%zP4Ku$7o^1$d$|5KO%3cjo(vNy6nYkmfAG#1&`Nj|L&&4;&wvnXto66_tMPD zdhgji$8?Uv(m!{D-qVb~#lv#$T>Xg5czXGv3m(~^Pn*W_r{CRUC%T=b> zRS4YUcS9Ama?p(Jg+gNR(S?)gs^U_m#$a;b;0l1)Mib(lqo&tEAcnpYOxe5t=Zq)i zgw`Y!OV3^_Poz#4_x?Gb^w8Mg1>lm{{soU0_`n(;;M6)69ba$3rit)t|0dMU>hNN{ z5%S&#&|Vw8v3D85c8!U%RBim3%!+WFnKyZ<%b2T7`Ud-fI2G*hWUV!@kp!gx-a^>b(CG>>Sm5 zp4p%^^ETi1tY;(z041I(wvaRNnl4YWl-O*_QIdy#?!uLm97W+vYl-(>C0pPD?+i3N z;j+M%=X2rF_GuL)YcHc(84p0*GJTYci|+E69UGRFdxPnopdH{h2S?-wq7|TkA@e?F zu61NRo-ft!zH8%-bId>AA|0rwO=NO~Y;yg^)Ogw}L+n0?ZhoYF-!o*#D3i3>P8R42 zocBy-;{W7yF*MBvmP0A(fVwy$pK2~Oa2o^6J!2Gh!}o*az$Bn&cB#h=Cd(HK@hx-+ zS}tkRa8wl&%EU1?!Vwf9qiuAimI{hi0L-8$Ar9|!31(D0swiS;x2<@IMn%l7Us4u< z9uDJpu5(EAo{tajAiY`Rn0vSqZXD}VB!okguQ zxnkzwA={%e$39)R1(Z&LIAldPmMosCbSFNNuv$SiW|-B)o#i0!`2a!Hvbr|DH$>!! zp~4hkv(`l?JCJ;7uUt^+eJ7h8B!bMaSQRPpIy;2+Csn++0{|!Z#gFCddNB;C_W?UC zEOo0F9|Df2u!5kL1C3Y9yLlKDZo3aGR0 zq=xMT{cjv`n6AoXX*kKXM)USVF<+iN0Q!`OhX*GudJK@wcISRQ#*d5bZ>54w|j%EY;4G=sQup%>Mdg0d9|7%{5TK*4-c>M`?uto(*IOf zw2t&*E5gk9Q>i%Mi7!tKq{gFby>2yz9;5P_hD3$;gVHoXK&iWa20n#YQDDFyabM8O zH;`z?7Ou;)93-V zMNWc!SR-8*Z~0(`4*Ha@J&^&#vktH+a+C%}Ag%_apSVbpqOG<$sl|-Tb*;)Iln!k# z|J7Q#*<}-vOyd9jmQ^(mS)r8G&0X6ZfwUS>L@r!5m#;Wln@pg8dmM3!WCx|4|EZwuhJ<4D z(s(JC@=Kh4h6*2hOukP4%lAdk9y>`Vu&kymZ;dct!Ga0PX_C!L6cND^%tRXRPe%h6 zbb;BD;ud)gEZ_0)EuU7rSkW@lnI!{({6lLU+mfQDrJz0x(ZAB}i5}W^JXST@1vd6w zJGN(^nKRiJH&5-4E*}pAqy?Rm-^^N~!>=~82HY(OdU_p!J4lT$FKG&#IAv8#^LmbcT2WgE5ix>}?rmbTf&CgWt6DLOw~qJD{&Ecq1~wqT zT95^LnkKd;R&EG5pC2zCY}-y~S+WZY3v+tTJAyts%b_ETZl3pspsk_n<^@w)(h0); z@(KcuD1O6o?sD9YrrncsStU#emW9Rb*jbu^N(G=Hs42%v+66=7C(b(j9aiuQY-_in zlRZT$=Se5serImE`R4hYHrO5dT7i+VOJYZfdN$04w#>eZWeXxf`zD7-b^wL5{M zDlcMA`IbVPuik=an`)M;$qJYMD;EG}h1V65fL3cyN6iu|Vuc=n4*9>SPqO4&Muoa|#rx2qJL<~bX%J;ph8cDo*8PMr~dx|PyW`C;1l{t|I)jyLX;dDWTl|27s^2Y7Nrx%y70 zj?SlY`w-Uc{y6y!6Y0axUaR=hWHNylgPeml%Hj&lfKw&yxAFCp#3P<5b$WUAm{qBZ zsVVN6%9KGz?7pRamc0lUtex+PLX6@4LcvM)5|gOjKx zNmM`YInr*4jxAlrM0>oKGfV4}Ncjmo=q?DFP;1Y)3)gCI33BOdW8g`mxR@%6xg`tI5-&s%*snHkS_Urw3_vZmT z^|^NE?FQ@G8+(e*{fgyE#~z8!D`V0x4&E`6gg2C{sv!*k$l|9fqvO8Umc3ahjAxvn zJvp{?%QNrKksGNAyP+1($Rn3&)q|7gSyB`3xO}@o3p?a-QQ(P$mlqM zio60#)qsOP(d2XMh*U=(s%$}vCyms0nfmj~u&1a5y^o_+Oxh}jk^&g`(j@~{UMl=( zTB!EyD)wggyj$5!Df>x2aE+;DW#xTk3iwo`{@$+XFre%M9xLvc!HclF#DerH4tIhfXiwIY&4u@Fm$cj>qy;RY^Bo$ckBYC_Jd+-o*}k@LcT$PL)gY4TIMEyA)9E}Tri|Xy*D(l-X?5_S1 zQqe|QOQ@>P=)smE{hnoJW~R5@?cJ(HpD0C!$L;iQeEhs*QavUlGZR-zN(uxT5wYZV z#h;#X@THof%F4(fSPUg??I@wVT=^Wnf#G4`^8Pnusa!gm$yvYTBY=+okCKUkl7esG z{DbrfSE}SBDe*{+YNI76tEAXK1>g9w-1&lEJ8JV#PyyQvIY*?1$Pg^94zd7-_vV^KulPMC?87FtckdyD^x1Hh$;BX%F=1_Gq6wtXc<2ghfl-ld#QR zFO*G*e1;z^E|9n>2ukgF(}DeZe3&f3(6&**vTk?27z5Qdt#9P2>PBhIZXJh`VQT*+ z501U4(!^06vqs)GI{HlL4_)^3Tq3L7vKw^R_22=|_5RU$yMY<4p68g-FP{XVp)eNc zExHrWPr41yVR)xVMs@BtRi5#T@ic-Bes&h!@^*~P~jUUqrSp1)_wH~ql* zcD|L0RpA#yFZwPYT`H6QAesQvh&xUfJ4BqNO0sQJOa^{l{P)>KFE(4Mbp-Ui*^n9c ze~&U+FR}8??P?9u) z#x_N=Y%%b?cjUOm%xXDQl%3wtpj?XTT`t)z0wM43Jmu#{H4Cc=18ewFM=xw$l>XR? zsy3zsu@pCm7X9n6Cg7%oS#i9s#>{Jt z$sqvoEBy$gU)?n=_%PfC~7k&GaI$g_>%=QkP?LpUBC7FTXN zv^=8I7|l*FhAx(f50TSbtKx{=UiAUb-rXpxQ;!XkRxlVmin#zmwDap zhD>NW`e9d~E{Cb``SgGWiiwZYXWdwH_WOg0Q`Wc=66eBN$u8q}VYz7Y6L%f#xF{r~ z^wuwKJCc@Qrj(I=hUiQm=wHSG5Bw)3PiLLjZTKY;idnR|Lv)GGwf^xmgKZiFtm>xr zZyf@6%`M@dw|SEC0BgurA9=XNeyVzKU^Vl(PjNqpo7cRQy4`XRkE(Z!L`j&1`Oc5J zr4@5gwCym+WlsZ}i&Ohg*VQ^)V2jp#c=s-q8+L$Y_0eRfrjOycUC z!77#rh#ck;I-Q=O0~i+a@}V4Wk=d)%N0YnSSlf74E)f#>kYCu0<_|u~bf}yU@HU3h z;X63fbR||-mBuV^!H1soHV!hd7Hdd|FfOxfu*dFKn~!kVFZd**7K7Fg!JGVZ3873< zj*mMZM4GImjR-GR-taQyqDw>_QWpk35$_j9Z>?GU#C3eY4F84Ds&!>$vY6Z<7T=jq z2`X_!?liBLFncJtYCvL5*F@y9BjJ`0VeiF=Aq0pA+Dc%?#VM8snTY>(KNXyN${s2| z{mxyb#__7VwJDbGe}1AE^V?LN!qQV$>-mu;7Bu$KKvy%qK3GGFFIaJZA|Q z83BQDsoMRQJT{JNh%T=5a>5@`X6X!@Kn|BDYvyQ9UXy%zvewNe-JD%`GvHGXd+y65 z5kT0Fv=!E&lN!C{O5TuSePpJOE-LQIe7&GQ$6jT8XWhbNuV!md@G%W20KLxTgxp?< zhu&o^!x|o`D9g?LGq^tu4C%flGBYDKJr~k(+qGslLjM-KUJKzarvaZrwQQOCwre`n zRvvx%{XcJ(^5TW{w4k_KqD)q+H_?pn3MSaHL1Y=VA!CX~3&ZxeYjzK|#HP^L?UyyD znX0q014!Pf%Cb$t3_XC*Ui?bN~{d3(W&CjUNNiKC!s;1 z#nXM(pr?cOug#xqUtVRXv*tx-3qJKUcq3qe(9iS4JyKz;Ph+&dydJR1OjS03H%1;U z@Bmd%b@}x#4m)6@Fs|1VEIY$jB>N?byNv}id0RxoaP#dcl&&JK3lBpyfrM4+&mS|? zs3)6%WQLX_B|F}T&U)&%p|dioX%d!uaCjjMje);ARmue@3nETelEcupE*JC^9M7yK z@~#g+V<%w+Y9>*U9SRJp37&~vB^GakkLtBd$&X(~C!Ip48zji{*(%}BKI78z91c&t z@2QNfG>ZQGXBvrl4+Jc5>a`pW0*${DvY9i|2&Tv`jmYR{C$q1Q_BiYDyKGs7hh+5? zc=)xP$u72P0}gTK%3QwvOL}P)6A88aN7SigsvtzO1@n^E=tRs>9Z3gD%8CB- zUE2yGM+}G4hLMMu8zrxEJea4FdY1(N9uI$~-;{j5Pu@8Wp*a!X^!)Adf-_dwy$Zrn z7Cs6eEGszDbndUBlf~|R#32Gijaj=>^dQ3{rRXsG?Gg(Dwe*$gU1|3~kb922NSt}6 z_afF^L&Hj&F<|?eA@3c)(h6vfEYkHJZNZtNst^4;@V$)_HPcwj zIgpD9#_`(me#u>J{o~lSCTv;zi4W!6#Ksy<&W)J$k1j!Mo_@j)mm{jlbaa|a?kHNe zQlkrgzBI+%qcv^;f`HleQZSJY^5%wyfw@~4M7AhIz4=Cy*>SwC{X1#dLPd&<^DGSe zlXyDP62gWii44Msfal+mbHz%gx#6BCft~HNnMySEQ{`8D8o{)osT?}JqS11FO)Va@ z$T5bX(z(Q0y=xsq&siqJNW@92hzC60aBd+|qQj4ruJ}OBr8?thojVH=;=Oj3qoXjV zC`N4}^D)8RFWJhC+N=aqT24SNd}!YBOx?x|Lww)o`buVdL_tu}iWWYahpxt;bMAJj zJ?ZcZVN0^(&U;j$&2(<7AR(hZVT*isG&@o7cGa<#oE(C&v2kdAXig3ReoTmRiQK1A z8(m1hpoImsO9!5xTo(84<>hBb*Q8BL#&@7=Zf*`_#-b?_n+>MwsfWhz_|}b*o_@%{ zyn3@*h1&NyUJvCzspfm>9&DgOn#ee)6YZH^*pnq(4vq0;->Nv%Eu&?R_P`Np(nvnk z{nksz$=VvQ6FFbuHDQJ*4sY9MVrkym&Bp-6=aE9K<&NhUgSyz!DWnieAXjXAY66w{ z!U)^|dE?Q{m3C$NBe{tvh@(ztnpuuXNld{Wh;G*xDM8^9P zegnrS?;n5jsdG4+Lt#U81kGhS)6orgD{iH53KO)~(+aM{3{jV+$iG(rBo`YZ?{NdU zl%+gHKEJyWsijLAQ4AD$*QWSJYftr?FLP`(|4!h6SwJH3Q@au~2c7Bea)iXHJ8p?B zZB`|Q_!?k|`l%{wQZvV6r>jdVKYD^8EmJWAZOm3!`}-WqT8XP(dt_S%9S3dtjV?!Nr2lQjONw z1w%@0Piyu`ob}4>3fiuAb4m9-3blb;==2&R&!I z@TmuohS&9602F|3rFjE6-N_#jNfI=|pFMhT)w^PGzuIQhn~5Xx8_U5#9(^ZD_I zL5Td(Q${;#Wx45cfa}Y%DM2j_AY!a#`zRaP+Ba`B-*0Fp{ubC8b^RSKcwSvJvVIGv zh`yw_EY?i*+Q)n&YciW!Q1X~>pX37DUL8q8^?b1z1vO!i*flB=IG$Cs(n%B_i&wGG zCeCm0?<3(E;j#Zq@|x87euN5VW-KI7nD@a6yX6(%vh>Gso~6d_dIkRkN50{#K-K-S zBJF}lG5YD=6${n0uM_d*VEQZN;E^E*c~JW)^QYhi9vg!Pjq~g)0n)qi=Q1M zG#@HMTJ#)W80(MX&_*I{Vlm@dvvl9Kw6sD5C$Y+pM5Ahx-aA(6EbKSnIqFx^M%WHP z{vxL!so6 zyk3?VFQ3g!hKMs9q1OD+;TN6yU>*vOBA&2>W>ai`##5YYyWiehL#4dAcxpb`@G`C5 zRee^b^_|Iv-8os>U*boIj?~8^O)pQa-oNko1_nLNrRVqR{6JJbiB)0su8>@5M00kQ z5RNIsqdvGk&zQZX5L-c82E*sE)rm|%NQujJ!A;UBFLxP5H}#L383mWbAQl~yC3YO) zlFf1Ng7)1@YQyET)lPsT+bjF@^afnqE&+<1Q7QiuV0b0bBeHB$h*4@81~?-S=DPuc z??V?%znQ{l+pDy6%_rq7jILg$d*7FrWrf8P1%=<4(0eJQ036G(Pxo_sd$cs(gD4JM~sQv zyPXp@*6pZp7!+?)iQSqN>n}XxugoG5tl~Kmdd#HUe;md9sABFnue(zDgyKcB`mArc zzk$zBsdJbz+qWF^u8THH_*&ETcnPN{tMGwmT7D}^BUcOk;LdF!1DMykPO0v=F4S^| zN{66rvlzZn;S~kwTA6a1gXIACQX%p9g}g&i05L=w#-+HHe9kv*o#Y&SW22J6oC~Y= zfyb5CoGR`|fRwoqQi;9ooESyd;Ew7FftuV^d3VaSlmUSejHaq3+k8hH&_5S@hYg4h zwQ%BwXx&_?l;!;5C&N61VdeXgx&Ufp$+YV3+?pU7Yjxg3DkTi^N&XqgWX?sqsX1`k zMdS9mD_j`Wkw%q3&fI9q+pPf+kV~F#y#c`cgmu9+B1T}v~rO3hEK;^AI2Jv9icjFlBUMzA5EIN8oci9ODHF+3r&E$C1gxb zbbY|IWX%#1+Wo$pzZgde;-$gjn5}xP9XdrhE~zN%&Yc!vVSdQk2Hu1RP?$aC&2hhR z{Y6}eF*<+K>u#uG!o!^ds(#%4`tkh~t@1tSB@VHRd~0Y$JLJ_tnjL+Zon?R3irGcR z=!}koxzd$FC{il6>;^Vs@H(^0(2}>mdvg2svG3LE`BNXa_*bgO)!w}Aal@GeTfX>! z#&3bQQ)MGOKB2lpysyugFK{!oP<)El_2avZ1QxDSo>UB7_jLa%_)I`;aEq>cl)TQC zbl&8@;h1Pbf#1WUk=B(Ob)m1)EedKB1-^Se$&*}@Q(Y940LS$5K5llSt*+-MHTzb*$3_cgKcmA)>byxk82HzkqXzTnjW+8=t-e7Dtp&(@O;19#HK;=7dc`5N z?;a0%z2&R8SI%bqF}X{AQ_t5}kpcv|zCE4{0!7WT0G2*W{)>-HdssSz4lu zyyK6I?zHLTOzysS`Tcgy_jpVj0eUb&9ElG4td*t*ROadC!(A##@OLv`)y*#3ZN9Vf zbN?e9lIyN#d@;j@tWV+GgFe5o^-FSoeu9$up8_fL+F$3)YW%~9f~m#D!8u<4x9SNF?xCpUb_H5((TTVYOc??Lsj z*tXnJ-m$6~dT+kJV&haLUq|u0xIV$nag;E*ZcS{mIWzw%?8m6aKyU@`s6j?&h-u4_ zi>Kfh^VxD_$arJjj;uRb;)Ry5r?BaEPgnl-crodXH17|SA1c+@0FKPFWZZL3JS05aV8eFx0 z8ryVN#9nTj?uB0nA2Y(P^XvBaO4();Y~`d9GW^T9A_2y9J;T-P@ooY6J4!enw_s5$ z{LJ#7w=#P5_I379TzR53aVg~X;yG>X5r;70FlC0FNCo^(Lh})*OP{NiU40oy%l{(( zwf%b4UDCP{n_F$*dFQ>K*kEy-287ZXcXl+%#^qzj#BNQvi^yBcVZZY0_LkHr>|%2z zzHYV`blCZ^@e>H|Vhs*Jz+&xHwt<6heZI`28JXUnCf|?5g9cV8<1GH6S;?sq)FIP=e^iGD{H4XJ*}xaG9fG}?MSHb5{+QBH2l83c~({i~*v$?WnAa+M)v zpGg|$lKJ7S$J=n^OroIvue}JT)E>rU;DRR&YfX%bNS)Kwi-Ny$OFZdVW(Yw#HzS9F z7=I#zBdJLfF+S5fwo4o~6?+p*SkKmE6HCyq-p_g4w(u$aT=L08C9wF;PmYk|5{?J9 z`o{vEN&U0e^fsSXjQw2%@7G9{lcTvTrRP-Uzp=-T)$F8+wVY-qeK)k(Asm&NGN->W zojEOA^?ANPSeIg*+^fEyf4m%`13TDRT-JE6+APUG`*`W+6GJVu!1hx;roGX!xCMO- z_{dJhxiA?6Jas;y0D;<5D>M$HW82>9 z7y{Itego;-tJ{FSR=g9nrUMqTY~`N1R* zG%Td;y=NdCPr0UmcA;nSRu0stZV%HrH#cp87kr9h&v7i6zph0I_}Px5v-e6K5cGQ9 zaR2yx-~+d`Ck|5h55w95?f<~~A;RV;DR}V(BKmMUPX;tAr`JXJepob4MDK$HfFsot|lGz<#DgF|7r3NU>b_W7%s^V9Ec~Q zk(8q;ETc}jlm<%LD^fMPZW(FFU*qH~2ZWCL5>9vC|8VBqzsxJjyWpijB(L*)W^c}+ z?Squ4cT#qxf)NVQ){9Pd#N~CsZzvt?Uk|BBiHY~yN^tLfr?i5h6T6%Y%8&}=04wH` zXu$`=YA^(hC)1i9&*Gk)MXdC^p@zeTn}3~WOc4B<8Wf}7jjt&fT%7#0$kaHL>r34o z@ahQ4&x8!Wb1Y*Nc2x#p$@UKUdL-7_x{`P_&g66V(+=xcttUz1&WKE)wI^ezA|wn4 z4agnX9BOG3qnLO%#B9V z!O7}>y?w+v%E3jB7My;`LZPXI4}NC zkI~qVFiTY-1S;M73iq2b?BDqBqy1L5i@NFSF97opeddIGd@x*{1$AJVQczQnlg|oL zhKg1o=Ylr@l4*+b{aPWj>m&&U<6u?C1|et6#A5b){KlY4DGvZ@@|vy+A4b)c>~ zw1Za^3MC(CX5Lfet7I^qh#ahmec$kjC4HTSll77%VrMtbzB=6)d)DLqQOH?`NL(Mh^rI^d+P!-0-NXuqEZL`2LSg#2*3PU z{c&o0)h&fvS-ETzXPZnsId~)q-2+e~Gjj4B9NDpv9X_OImV*5z!RHJ3h9LvDB$Id>bu({KC{G=Jy&ttY6dEgF)!;R13NsGxAnC*=j_lY=?h|A2)aVI` zAHjS#7b?<^^JQ`+xrYwW`<{n*`@VRstEaxx31 z^=W%Q;0V8u7fZ9rdSx-WRteEy!gwVXBq&j7MOqvthRs~Sj4{1%*NNA)BS!CTjJ@MQ z@{jK2?4o0|m>n4OYD7_i&c&BLWAEaj89O-iaU!x(kGGExE;{9Ho9|myPf05kJHf5fI^`lAt8Zb!-oChXY%v&Ieq#xvMl4`;zCqZ6k2V2>_?Vm ztX3;kRaKOgm7!EB@%Q(~$;s&#=a)1!HF4s^2^@}A93?I;?v|1;#l^*>r?)!vN~MzU z@NmMy!n$l_mvkjbx};04>oO>nXf+y?fLV56HCu4VpwYO|XVTqFU%!KolTNT{$1$e# z4JF8_Z97R;O3oy5veuhfBM0H7Q=-uM5auKj9^{KbC8G9-WWs`bm>T2TN~N4+O`O+`=E&o>h>iEFVvt59l)%4vxxW7wYL8|^-5$Y<7 zC@pV74sk)LHsKp=L=zc`tCJRLuGW>Mrii3fOW0ZD#>p544r2m%gu5HlZmm^(*AVpIws#}VwQq$KYdmL4wRz>b}qTsWEFcn@Sj zL~C%u%Q+NZ(`6o4m6IQSZd%A}rnbHwi!9)(btNFo8EbeTE(R5XY$BrHc&3l)kCUP| zaec?)P#Q3*&vBu!oXRpQhmunn>)#)DgBnp~A~sp19NdpNvnI3c(5K{|+{V_U3y2sUbcq|;UYkeqfmB@j%wtHXdsl+w*O(`94rJ6_ zFW?gv!?$02#ZS9WP+VI{+JT?RJ$sC#9XomOiN}~dG9GV}s&iq!EQ0_VgBt-s{T#>1q?QD&yPs-{yQgs9Zx6%>lDF&tkH0~A~h^;+vPS+{s8sa7A}96g*6XU(Pi zG)DAe+^_@+93}YsdR#MC}ZqThdBM zN!-e*J1627(aJ}xHMkKN6oB)!`ZDTpRJ62GG$5dM@+8Ehr93PY7p*{doLua~aX!7coAD{u3;yO-|@l0?nEpgnOwG zGOp2@Y$bQk8kTH2jN~_)sgwJ(B}iKD+u#$!pgys}VIe+f6zLS7ImY3$IgE*OrY3VA-!EBDx%U)ikLiu~rNCQ2?d(g!xWNQ0 zKFIlBTJ=UJ++CeeW2LSzjhqr2@usej;J+cVBw=?rP^emI4h5wa-^l*V@sD7@h%x;1 z?UyWCwUeB(I?nE0!{^`jC;o5u64D9!KvAK=)i(&CB%9qU7PEcdS#0jz88UDL{WH&y zRZ#-B00(zOsUg*sQe4%HT{d)lO+kUiV5IZJ6%H&G3AMVFhh7lXc!c(1`u*MLKXeQ~ zFaL(Gmu)09zlQUN*74b*c=~mJoLFyNC(bGXV^{>?9!?}?m($$bh}i<@73kgk@Dqzk z-Sq>Tx9_LO?m@qygBac~gVgM*jvwupzES1EfT{QKSXu%9`Z0;Ly=%!inarlu1DJW& z1KcxfBw>bYt|ri_*#R8}@}>YrjGsZj4Tsp9x``im-bwF;eNbD=+4plYEkVN=)yJz- zb221JqNT;s`pBY)QV{-6;C{KOjNHO9nk+J|ZO`9+EgK09^+l}^sjaQX+E9nh(!_4$~Tfh+1btebq8lK^SbC1c6^q%n{|wEha0U+^N5YwdoGPyQPf&Lh3N1pV6@=}&TrFX*H#|K|8x3>KzpfZ^7^!7k4pQEax5__kT)GA!O{4l5;M2EN|aBY&r zPF~V#mTo(Pq2Cis7!u5-<3a6CT>lXa>UWZKDHsnYZRhl&&=Hz2gV){OiKtO z#JLTpk!+Nn-OceTPo|6-ijPjb;TffM4PeORNyMx@PWF-A94fez2(JKC0w}ev^d3Kw zh(&ujo|?kJv|Rdy_Gn9nNR%Jn!x>9A7L4n0TN5M~6A93SOx?tahU5V zE;z@jLx;()wt=j~ekK37ARuZqs09H*ZN%N#cvYpFsP@3qRgEBs=uOV8v;o3phu_u3 z36)aBZnt8wTn$jFR4SD1S{WVGRi?2wDHV(7Sfc$rt{QHvG6ZtRtDiIQfm*cget5g7 zIvbXwHn`Jc%w7B`wjU!_E#bQ#H*p{}hw7>V4*$G}tn@T8UwngS7mmcw*aqqf0!oDv z1Oexu?!<<-Qm|aqvsc6j=1uL(#?%t3&ZTp@sGJGCdJ z{;C~x?XV@3o6uV#jOR)3M293rZRTrCO+SLhRtiYFfHK`!i{2U zwQ}YOk~3t6zStLUle%kz|5_6z1`m3UzKcJ__GiRTOIY&bCib7mq@uQ%S&=A;XE^mI|OM_QDu|Y0KmU>DGa!F4)K}lT;lBCCOx8D>rr&K6W zDU}FU*2Bv(joEwIakvm&U_3zqeq71ir*MhmvA=!8l(I&Q-T`=b+RqhGYMcm9n9A=W zdNFLo4}8094M|7SD6K8y^q$qcot;7E`B(V;<1+|%)3yTtf&zs?KoB%|`gbF$yWgb& z+k!$v;DD*j9{vOSbCM~|&!I5C7^1on)bCE-{J@_k@7}}&IO8y1)>m1*8A(=RcU-A2 z5rwEkAquFCA@qz7M5_b=4JMb?hm;J?czL+tC_BQ(ufIb5g@>6rW*{+PzPN?xz5Vdk z-zZvh(9%*$NqI8}0!Alil)pIGQ9z+}C3^T={utAjVQX7w!0bPnL0L^HCwDF9t(8wxpNLYSKonIN-NT8F_Gmrt)jC|EIDhnw-xY~9S-cWgk)}}vb>xI30%eN0f_Q10xGSTQYnY` z6w-ZYZ@l#C%llnzBp`YS_y4{-vC(0?@yCzYlT*#v#9bUMoJ*uvV5iuP1dR)BCN+X6 zqExC;C;{0)W%ho)`{X0mr@@%J9^tiVg9&uDuqorGTP!9KG5YuA(Kr7|c-;4Vw{jyX zCo@Ptu#xlWCpn(>EN}kq0r~|x-5xp=K}B%?kxUxai$lwfvSZb1=8lggwt62sPBqi( z{z-H*c8-EX5Jc4KHXGz{U~B8JxLxZ;mTj0@B-Az=R;wKe39F@^h6agikv&0B;Nt9r zLJ&X{K@^c?J2r=mY^$TRv<|aXhK}ay$!+&72uQXj@{exj;}5>%ge`)34?M}Nk#RUt zNac##3x+9`svEBJF8Lowl&JLhM-S(*KSVQl#2OZTyMpcePf$=a^;n4(Np~x=JLiXPcXc@$0h$;6h%ZqM5!|p65F>eLDFi!RZgLdoivHwYj;z8 zwvO|uC&)KXAx`JZ^r!xa)2Mo)1`fqTbkI~)LT>gMj-Jk?$sr@z?bxqKItqe-Ql$ix z1V+UXcB>sk0jc6NTXr4e&Y|52byb6{lH{c0sN-fbD8O*r_e*+u zI>p7s0QmU$(6eXHU+mhDWtsH!befu)0Emi;!q?Zg-QU&L*7D_-Us73F$?tymJ3Ku- zIe73OpMU;2Z@&2^d3kxf^2#f?xw*CbY(+%{R;%^Wx`>E~xaoN$4Gj(K+qaMU`ua=H zcB2lNF6l~=bV--|>c!UBNL5u078%qUJwATEmo|)`ai!O&nT+eRnZ<|q^255rjESC# zmrhATP7*s#n~8sHJl%}fIQA*g;j9?4Y?Ki|isZ z$`Bn|m5OVhTI1wRxYMn1JPM);7r$8M{Prydjh)PfAD6Loldbs#JmwXVlfrvr<3DRt#vZ0#NiY9D!FdDoWJY_zzqnW%@ds&}+kh3{u zSVbFEW(NYQ_7#PKaD}7QfeWPYVZMn*J-d!iCTMsd3Uf7Q4)3Mhqc1~ag3&3u_V=$^1O+Ca zC}uwKM+S_V#QK%r^TVpm9L_AJD(?vYe*dqy`UUag%(xrO^k@V3oB|m!a{;oYj;s^g zSiS!+=>^rujxuVRY`284v{u$$`$387>}>KItti!6G+I@=phf5EOQh?q>&4&}!q|IW zqR+^QY+1F8r9Z4@@0mgxic(qj;a@QN1@YE`1YER=t8IYLx_7U}uDlu0KL*qCL)13b zVXd#bq_2^)79d$_$v&}_eaVMOyHEp;QmUJ69oqC(g|yl44ghe`HH?WfXR<3Llig>x z^ZxISvN2&aQ>RU1#?sBmf*~;~# zWRy^ydyMb?@i#nt19)L}A3E0{(RL;av?af;zc4$hEVY*8b2{@JwRRDshZoKrpejPU zb6>{Zl1p}OBN~q$jGi{1VPV?q0->&cfoQE&oJu)HwM{{|n;SZfs>6G{@wQc=)u>U3 zGIm=tjSbC!OP6i@KNlHu%?0*s{h8ig@%XlT-y&MqU?$x2BVq)@`6EGK%pb*UZ8Y1k?sk2F!SbXLF8l=Ix)Utt#e1O*5Tm zH)^HL5>$HJy}WVOLUZ{^mb~*S?`?17-nTyHsf9yv*D64&zU9si99EiIEGV49nDX#z z44E*Ut!q}YZ0V0Ao-Sa=k`M6;isZd#C*i5OP4B{~8&l^@}2vP;7j-H{Sb~N5T*LD)gG+fB$oJB`Oh!1t?yLk82 zS6N#c#h*TVfd@x)LnR8>nnnKee1?R>ZpLDj5kw8nZceS|YM1=~zz=Tis_j4RH^d~2 zA!*Z(e7|%B+fy>Asmo{Ew|__D9>^yzO~Y4z&)M zmM&YtroE@AtSe^UkDv06lP3Xxe2DJu?GrenQhCjO6mahzO>D3)JI`d%R8vM(qlBPS z;S)QO`7sU}%FmOuY6bg`Wl`N?q4GlO@Nxn98dvzD57idiU%S-=HU3e`X@88c?I z_qnasOBj_>sl>_2>89q9$g)gMP7YhPY(bV~R4NtIr%xv?F0RWqc1fEz-X&eqB^`*J zin2nob4zd_Vsh(7&z^o)Y#0&O(B4d$ItE{LJx4eG!0zliWT~FRTX$39+=od+LbxqQ zm1LvpdNIpoadgrpFV+x|L;{LN!S`2XtGOK8|$ezTao^w3(AE{vJ~D9=9oAVWi(Z_$Yo2^cVkx88Y&XXcM0&}bv)@Xvhm z_V0P}w=eL|mHVk^k#66+5OhS1pT~q=-ZbSOW5ed%6xQ3RFUw)y{&RF6GK_A%#xAzx zU!j<5Dk&_jz#?BAQ253S<)N4V%BLT{!=qDssG?cCawB^y<_M_Kgl zE4=#tQZjA+OkVgnFFmz@UV$#R=x7KXSO%G<#zt%oJLcvFnwu^Ev0fbZYVvYRv0h0! zFRE}0?!)}wzRo{C`;-^v55YsLz*3dY`ehr)s%h?&jS~bBSASoeR7wOHE)v_zl9kHy zM_BaEE4=dlGO`?jOu6TAUi|HRdimd?z61l&WAEcHAH2uIGX~|MScI`eYY!Z!y2iShNnz-TPh;-|0^>S-4+#0w@ z*98%z`eL#R>#$uhEnQHk@rW42eJ}i(Pe1#B-%js?s}>pyPqKFXcJeA)xb5U8?JEUn zuB)Z0x(-KcSl{{eXKAFN#evnKz}-KL_};zg-O>N}I6}Of5bRYH7B?WZRx$iX577!T z2$urJF8MEsY^NyuG^rVR*ButEa1NqpbT?cLO4J$?#tyqbuLh7LM-zwFFJt4e3Vfsb z(zi!P1rhSae;r-!=DZt27;%xFsKsW|Q+BcQKqk%C7)EYZ@Wx1Z%9BO`b4d8wg-vRX4L7cZjX;_wvhlpS8rcN>o* z`NlA~ZyatHliybw!bXfE-b2@EpHHDy;}a5$vj|c{Emc+< zx64Ddo69+{aT^&eY82W4di99F)gVw`p3kY188pk+f><^xDywmzCo-lNfx+%*w64TN zhoBMcG3T$QIK0m!v$>wM zLn&0d_h3k*7s}g=?AF!QUFr(aXf#AdM&j!F%av$ZSy>bp7bD9uUS3{AMMbqsVA$<; z($dn%&CTVDFTUW^sZ(tbjmVH8LkJ5Cqq@49ty{Nt_#HZ(?otBrVnOW9NMmCoix)2@ zD=P~@5J*T!VBx}rT~u3La>efMk}m0z_Q8~;maG#=oH*A6N&{hoCNMnQc=bW1@nFb= zDGZMGqV9YOYu4_gxZoJu4;2wMd?GzOwYTF=sLtNa2e1B)4>z5o@BHU@?y>od>f0Sp zqxydX>=p!+Iz3v2fW0)A(-$gec61y}G}P5%v)(9~;h?7YI2%?cQPRPms!cxh89SdB zUVe@#@tz2F3#DgIaIVqO>H8HGYSbzLLFI&piw>niqT#|ZzWmdR{CV*a0w+DfOHVz> z_WW@ap?tu=w-8a^HkLc&VGn*uS0+-g=v*d$O*huUiv?S1<0k zV>G_XI`(hgNb>1ia!>9gwa$kTL;K>U?h4?3g{U0W=TER{>mG_5q;^4tO7B6BQS*5D z_2-$`(+`!Sk)pgb3Q8=Uvr*@g`Q(k~dFQ9|^q=(BA-+I$Aif0E~(xQEgG zqVP0oZr#?%4rH|(y~Zryt$%*aHy^#uoMEw;?3J9{wSxEl@*Zna3b4ty?`=3oj$+=_ z{7xNN7Ux|$Y?{TA=of!*#vmO-IZqtka$ZT&axvAOUh>o;uW zbWsx>`0ET}f44HKipLm>o8n8)~U* zup$GJ-9mL~DV6oD-zC{Bn9Yq;*EM68+O8*pY;UHa$%fQ+y_$yBWUwsDmxjGdR}8pT zsJ$36VFW=gS{n0HSoGN!{Cp^js`@5cnwzMtC?sX$cdXi7Mqux5ozqKQF*{c*=TJdC z9p!XohlSeWQtBKE{Q4vi@9WmCOTuoo(9qa$4Y+To?9@pzD$V%C3}x)FNHnrUV_69m z771%}1J!j6IOMiHTvbX*RXsLIMv`n;%@(Sw>##edw*4*xGIqPv?ktcU7V?fHvE$I$ zONnWP#+iuzlX+myP+axutGguS)}&7Bb)wzem=reW#10#K)_5%BNZR1R-`6^rOXel8zy%5nooKfS)5$Uc% zsduKw=n3@jP-8AV$+sVU%H|{4RM$1q($Y*_3-2-+DHN^##)mxBc35VT=-P}Y)WkcKGb-8V=%dMd9^~|~5PIGxF)m9mePZT|Z zoKSXkYX8rQrM`mw8&+_*(ApjlQyB>AF`S2=f0@VbNWe+qpt1Zcsac)#A#8Q$S^LrJ z{QmDh;fR^db1yu}tTFuw^)jKn&0&|IQlnF=5Ueef<)m@GspC!Ru(wcGUDpXWzKp$V zHgKe%x}Dt>6dJs{^=IxAukqr-A-L&8nlGFq{X*Gw{FM$J9n>l%fqC9+A%0r3hoTE7*>SLd&|wql<$1H9yAuE2gBac` z6rnDkecSg@+)%`>ttn`FPh{q(K@1E+_{sD8#m%`IPmuN=JCfLr+fGAT}HA? zE}scqD(I3fX%G5UW*ubB%FX1N!8v3Q^Y5Nauu;`+`?$o6WX|~hxXYEKY+24Xi`H|x zC720gVp<1<|EJ=pV&~#7_uoZm8`w9NW5tua+kv#Z2LzDFX>`({)^-cvDNMhEs=ADQ}i=@{M5loG19l zSKsi?3-{1FKu=9hBC9qZrl!Gq`;5BAM9kQ^OzP=DV`dV|Rn*FGvMlc}St;^{D zN&U_PUJOpxVHsS zjdxT(#!eWFx1x3Evg0~Hc6)WnSnBF%wu75bAYT67NS0bsR(;CX8_(i9WE#^(^~2ZH zUSBs0?8}X%oI0IJjai9Da4Zw=d5O;!ea)YrnM-$fCB?@R*_3pY3iEBOp;Nmvc=kL7 z1equ~wU_nlzGca_OuPq8Vt{{Viv$#C_1^e+=@5`hrqI+ z(BKm}l-Xlr&?*FE$vs>x=@ zwj?r2o7+8`Y;Wf5o-J%Wc$Q|-9UpH43PGYcGmZS}TCUq5^h?bqvW2E*Gj@l7R_}qQ zS67nczbf`x4zB!!cRyZCa%v7$jhE?IY)w_1*|~}p+m0az4q@K?3y5=3UiPzUVd+Ye4jg3HrXTqDi=`YrSA)Z0qcZCd zKmE9bHHoLFZV}Mx-0^T#BempRb-k4AO6Wj&Uj@i=3RCOaHSo(>Zng zARCvw&+p&*k|NI@^z_goAm9=`fkz&gL6n!k>7A?je$@{0Yub_+)&{cntYOP(D}5*3 z&4S7C7!)8VjdU9@n}u_R;HkBevv(=Kd*TUR`RhOV^wSS{>*Z&7;oWsO^%+mUNOx39 zH6DS%xT~T3_$IztwuU_i4wAHaC7=Im8AtM~aL6r`oJ-~B<=?Y%XDVi!0~}4%);D3d zOIR%R)YVz9b0%4+%FU%pQW4Z|0C8R}T|DOh%bu`grZD?BC(m4`z(!DD4Cuv}F{21| z6_F(gd#4H>9ONY~<a5kBJhl|>G zCCYY+PHkoFy2G6@ywsgP%KkHDSUY@P0fT!8BWFz@!k|HxrPjoGhpluS{LtP^Lz5X< zslz8CngA^{oloSmPnU3{+Q`^h^BETHeQBoOt&5GSl5EbV6(Sfti0n6>r~mX3i$41c z3r9tvH5apg$2#_b#*nOC=wJDL~w9$yT4N^m5d%enq|wDv1G{-;^X5l z{jR*coQ8%5v|248At4=pkDs3(adB}dl}cn;=ER8;G&VMN>Y4KL^7!hjuP7=i!qd}} z=bwL`$&)9eQmML(WS3lCqg^WKl3#^mB+IS;I{_v(w)kX5_6^XKwiEKSG zhu(p`aq7UJ31aIoZ$Y8Q+dm9HFYQ%}PXKV(akQaUlHE>AOBJUMpPw3XVv$t9?+ZL zOo+nR7Kli;T8<@d=0K*J`==W_72MVorm}PU5ynT2!(G*J5Tw>>QHUa@Zm~qTX%U4h z?@+>Sw_&$md+u1sNzb8H))O5+keDzRtmgA=`V#1AGNQjmUp7fXmc@4Zx>6Y(ogPws z0Xxw}hZnw2qoTZqX0yzt77zLQp#R0WRu22cM1zdo z?m!e3muIgCA}-;hxNBN(_I{Aarmw%i;4zpt-%P++*?A6TgUN&NFn<&&=~S1O(c*Ai z58WG^Q+E#jl_0A7D$ z77-UIa%2aU*(q#JJVemsJ~-)|i677t=goOkXYXV6u2T#Q8;OTb*#Yih!z#5^L0_Bj zw=_~y+kjP4;pG`Ym|s^o@Lv-_#NZRbK$9m`2bS@A>TY6s_N05T7kY(_%Dgiq?nVn@eHL4pr{XI3sLM z=h(S4gsAud%$R>C^Bi^L{d))bN7l2#-<#K74#U-;Lh_4c)Y!r7&P!zbXRmSY;`PGa zDLT57#KfQ3m0F4<*~#0#n#HT#nJ{z^@e$q_yyAH1wYO*y{>nG&50HO+8=oKF2BI3z zZv9#C*h@Th*Fc;)RWu}^HU^L|M1vz^BY%BmACW!di4F6^sD_%tO!n{IO|~t9M_+q} zN2c_{P1D{6sDu_W_O9k1$|{ET4aZ=w;q<;O?8wn^$1~6H)SO-z6at{Yv-ec)oxF{A zR~{f~@n1N*<9i~adogOp0`8w@BR6j`S^0-qvBIAhUwM=ZCwFr!c?0`0Ymw|SXSaUC zXWw{e(oLSg@czB%;@HK z%`5@|oog`TW{+dpu@okb?vJm_%~mmsIQw;H*rm9{i&;5aB z$2)wnDv5$qyZ9<~7YIt+L;Er3!Ke7m+!1(b1dvTcjhV;X;fGk1n91^Yo@ejM2qI$o zGkxBj+#}_Y_Vr$}_HSUVnufe=eK2E2C*bR*@3L?I;)Q#?8&Q}fk z76d?Yw04!rtrVdSsgaCRrzxmxMD1(DUS(aSoa!;b=Rn zTdyafP%5wQrzw5uH=qZeYYvi?cZB8NEoDIDZ|M_YYDbcfbq9lf+Q<6z9eh#b zNx!>aXF!lUx5mDwC1&&l`UGxdV|Eh%`ml(eGag}Zm?ug>MOeachQux9l5Dq7TT=^q7o7Ds*R5``Sjf!Gq_VOSQ4|Rc4ZUvC;-Zw5m64H=fy3dz zWHQmSXHT@+j*F(WT5Y?eizG?x-@l){ygWieLYOscR)?Q87z{l4;Da1EaDejia*~pg zc;t~sFqur(?R1f4ncUo5{`}`Zvv1!%TwGju>ZzxA^wCFgc6RPEj$P7~BQS}*%v_Vh?#=sY^6kmJbEXpMtwU^5 zpwa123Nl4Uwz4yIB=Zu2kZN+-{^Jt%WLILB)YMj0QgG$~C#wvM9Tttl?m(7h>~;%{ zjm^lowyOwlsXWE@Wn1YRb~ll3Is^n%dS`;7yW_4+CL|&ljaGdrA=6@UAjuMTyA`|b z3hfJO^m-jCg+zUIF4;L%^bV+I=e|PXMh_+2{W2IR>fGovdKP1PZD+~ha^(K^FmXUY zR~N~zTpTs&toZl?x_Q3Nydm8%XjGT?l*39@#vXPY%R_LFVDyx!#JFEvF$y#qJsLIG zn+nLtETi7kz_FCGm_q%~=(UJ6k+wgPlrzI|>*;~HER(IvR&hM961&Zb`l=E#Pwb|| z7S7Nn3IT9 zuST!eBFaUa-IqwpnPIs0aHpj-gDuNeb27gghy9|yc2R7LU`P))91c5@1d^?u`bK+) zz;{D-5=+-c5En3wKo<=H0&2Y{u`xmD3>Aa~`C!xwmx4)))rKU=I2=~2SC%0v6=?K& zR7f=CXOLN5gTA(egXwkj9x#L;kITTDO6SJFnRhXCT?!ksYYCq+o)O(#ZYnTTJG&Fz zr#IfK(r7Nqqu6XC6kP}QO2*t+z}ip$L3i)hxocz;CbjaaHC&T_h|P&<;1SKVDWeGX z(q9G;^?FoFk>>hJ&S#&aHp+uDJCc$6PGIo=WADBL^s4H_|G)3k+w|UNGMV&DDygJU z5=>~Jh%`|^L_ms)%I>ntcURr7y6bP()vxQayPzu~3MeFy5J({*y-j*DJu{h-=`&9+ z_uk(hlV_Mo010v*F!Oo+May%z=iKu-{oJC#Ne^AuiH5xRf*N?vPhVGqgd!T~F~Z>} z@ilnGRm8GuSMu^(2iR0w!|(s!j~J8vB%fP64vz)GJq;W@UQ1sbye1y6q`w2yLEUq+`-}fEnFId2g@dbF! zS0QT zq>wUpDgNZKEWUgRlS?xP7YEP&Ue2tuzc+Cv3+GHHF9|wQv#FT5j3<72DH@=dthh7s zNb@)b!;i^I`q(8r{JlKpt=P);y+>&6S4k_L!2Cr^SvY$FS?&)v)Kn}P%=zp?Tt9CH zX^~z|pQ`1^p~KW1KSUrBL$N0_@78be$o#p?o?1bg!}Okgaz{oPD{uV*)2&T39;qUx zE7;4g<}qj5Sj^q!C@JHq52i3{ z@#V}JpH5GmmEgSV`G?AtgZ*EV8GBL|nJMm}6B~*%gSj_7#?LQ}5a?;*RC_%KI>2mo zbNQE_Wd8g)jLY?)pre?bq~}$z;tPMv;@bunPAzs1IoWC0&yG**X{Fr!wI7o=dk6Ks zCJHCbWZ@;#$V)Q6=R$HQsFOEs0Sl&A;Qk2J+T_Y+!IjrB@0DHb-?xJr@3myQ;~R$* zY|bPybMqNLcNO2MT#5E!uby;dQMsUucfTvFU{5XO>WBV;vGZ55eb-)&H?(v7WRQ&P z0&aZhIxd+#jgriBuaL2jQL&Kwzw;AjE#JoWU57Z`8zH4=BJ(a?#KO6i_EgWl1`!UnRTj67K%naoir$ z2TyOgQ^?LtKIeNQD*n!c>^#&<;jC3$wQwq_cG-yZsh9mC;o85R^CiG|*wI`eC z>MWzV;Q$T6d@f%)1*@hK3J=a)L_?vW4N72R+!Z%*?VOGLV*N4pzVZv6i1l#mjmwx) zn1r|aFza4}X`` z#s&U?>IFTN9 zt=~oAqD#4bQ8`Z2;20*oyn>SS6som+@(VMEo+}m$6N!ROgm6e5vG`)LxN)B?pPG}+ zbaj?+s{R0{A_ZKwcrq^K-B^}fzJ%MZy@bs_Sxc{;#p1;?$w{_-vt&=@Gndci_wQ75CSqd3{7WfLF~7&VK0wvF|K`WPe4R+~GQRdV z_cO0J(ugL4~QU0rP7zI`xFi-l>^rVW;JG>XMy8Cr~p#bQI7kx!jE#nGcj zfx+v!rcRwoBoaYUls|h?L)Ufc>gssmg%>cHOg!?)BV2OHCGS0fZLu(K-aM|o_FA5K z<{6G1JH~H*^BW#}>@kv)lSeF+=(IS$pVRx zMG`RzJ44X`J$?OjpQ_@WcMj991K~5g{oDT_xp68b`I$JB2)$>TI9|1n-TMyF(hn6^ zet}y*|9MtjHvP}ut8Z~+GH2B-%zJ4s^;0fq{**G#9lNQa0Xj~dJCP7O8M&9Qw4IW;Sl!mJW5y=)#YY(L3~{eR*+UuovWid<|a z2V=?$nK2=U>aC~Q_Uu3Ke_F5NOZVPGYU?Qu?>$OKL`CmD#epLym|Nl@FFgq@*iY-p z9UN-(5>tb;)E(!e$!VD+jzBcwBIz~sY zpGd$)?V-I?x8+lsm5RffMCG#2^VkE8{NO+S$nol}{MQc;^YSk}SQH(zD~tIzKFsYa zr;+qwqK=|qwOR4iZ{c76uA8bA*Rb-+g^bPdU{b@h9NWtC&pgA%whV6i>NmM}`6QgC z^DnFkHSgfLU;mz5=PD-Tcp%ipmgj%QODC*ceb1wOVfjQn7Qmgxg!z{-Gk-U`Yc}w` zNAF|pw+j=FR`|^GCbu@6`$Vs~T!vv!p?A!7tN2aZ$ zEH4AMb=ZlYa6fxq{wd#2HuLZuS5lm6Bh-14UE7XenRYFATyqKOZWEC}Ki$pysH*EA z7>Ur`R>P5lHH@EELPnAUlj5eVVj^XEZr*O(%8$PKMb;E#GIiy>EO4GXp=c&!+$CJK zd?6d__jBnr*DxVXj**UT3Rb6saPtm+`rU6*b=9?8yKDjFxyhK-5bf35`R&vH#n$!| zR^0P-?q4+n_s|x6JK6aaj4#h-|K8&~{lo+8tKQ`sMgRaH07*naR4Ab2l52Tz_FNWR zaT#OYYNC4oA9?b@MqV73hs9)P?2Hm7mt<4FznSg7`w>r^xt9AMnSjIYAiE&`d&hYC z+g};_y{|5G6YOr~*omXmxAdW^W=Os?Os;tJh7eMJ@*1RpTCN-bQ{tBCf5Jzmu&0G;-;@Y&TR`y zumYMrl{r^j#+Z%G)a-kSzk9HrKa9)6q}ZvLQOcx}Eb0%m@Xqr;;)(Wax&JFg^taV< z{OB>x1T-RQ9b4Ym&C>LF6lA62vb{Sn(!w3Q{mhT~u}kMmH_Rv3ZNcAmkPW+AC|U3s zZd^8pOq)V1?4_smCTs3$Omdc`G@_3m!#`k-z~-VI`@U+xU%E(XD;2Lwil|N!rl5J#oTIxK95|n zRYFCk!RgZ+w%HyG#T(Gp7Whrf*(QhbdmfAlhnHUr>$x~~f=uCnbYE`WznZ}Py~n;2 z4vm@sZ;#^Ok_37qz4Wxwc(&xoI#fjmsI;2qn+bBPf~F{UzPi+C)tR$d_Wj-aHSenI z+W5j9jITciZKe)+;$L_%UMj|l9sLKB+Fv>J|k_11odsEUiX^ahTZJ&f=E$faSlXgrx6sxqQUJl zuwUnLANP5n)OfzNBWi@np9p~elFXD6P3cXSp37wVWSnTgl-(ov217k$ESg2Xm(;rX_X?VacSJBxhQSG8$6~t{)ffY96gu{8);QG3Q#a_hO4sL1Alwly zo?qz!X}%oAU*d+K+YZszF?w5YRZc$a8Q0D-z{phH?X2W(^(tyq*>NUF z_IBuwpVejGu(tlD6J+>?Nv(@)EHTi+Jj?4kT~ztvx6 ziY4u<{0;jzR1A2B@zdES>WhfHC_i6FVFPG zCp`Ifq<94`=)&MJk$DAJdt8#`8O-uIwA4|yG+%;pG)em04OUZAgBc28J$b0d=G#XJ z8Eb9NHi^w58Cz#!$9i_P=gl`+Fz&zAo^eNpdv&Lp-dB!n2d%WJ6nHzben=?0d&|#z;UMg-m2IpoE>tbQP4UBM`+6$mnUpZ_SYjodW$mprseMF%8y22}0 z=u)6jo0>FYwJto23mL(=PG5NycGEa_-Hq^Y7OqKw^*GqNM`nbPr^D9v4Alpg)KuHN zc-wV;c6Wwrg-HBxuym3ZlIr%p88L&o2hx>PGC8 z9G*Pd#^fdfkFda#hTz+865_UM&kuw`i~@r_QE7n`&HJH?%{fprL9{7yZu`#XJqa`R z)uf+Abt(Pl@S!yEZY28*=3!@Ryztbo(n#XvKbxAqkFY^$kz;Ov`6&6jn9!fFb##?Z z*?C#G^ywj?;l3LtNBDm49~HVxQ|!o+Cw=HYPYbncsuUFG^*+kHW{&WJIs$K5Vp{I- zT4?Kk{mCWO-8|@VanYv1qZ1QCgGu6fzR*S0sVU&{O+XuRY`y}KOon!3U zMoY`0U;4AFIZy)GKIe}J3W&eP5=W*PW+9Wz5HJn!{X+6CF`zdE!Ew9Zn1r3xE-QjjTnko<}&%wRaz>&=~(1C&g|FUe$+d`3x&H7LzVI+|w!;QL_-twKuYsRT>>}X=zs_JEkBr z{N(Xb$G?F@E;odYv)l(!!!GV688i;Pfzp+{GN9Vl?r3q==*ju(P#MzSSN`S{R)4qg z2emPj@`@_~C<`BpTax0G5&WYswEkc~Tkm%8g9Zfc_=1LMi44l8cW$LA-usN>`}5-l!n3U3PG$I}WQ{6SQ#AToxMIuY+ziO{5iv{tirzJqWp~dc*>yHPtcLrugaOR~-R4fc2 z2{F3Jt4Lkv=0r1#mxjlUy6|&m7)M^rBHtkPGoIHB{4KtYWzEB-8)+{8zEV&R`?NZ# zdH&Rvm2~Q#!&_R@+D~a>RQ)(_UN~EI!XSonqzWn-+^NiVrrZON@2p{jW2!1YmUw~_pIlAXDZ9Y)4LAS z%C>@OgPs{}$(0U0Fx}XI<+5=}9~;h$ealhO>iO4ycz%4Yb$^BtG2PCJ%*-wl#wLxG_{+neG?E zl7H8Jp3AyJUaX;>yuMDYPQ{#fKaI|?V%_;HJN|lzIH!=Z7ZrpYW`jCOBiRRn{Qlz7 zHm@dWI-SDsT&dDb6>Eu=7oA?CeaEUTMqOF6uO1gg$iB)gXg{s&EM>*Iv*whq)pI3o zv#Dg9kHjoVm{GT;)m=?WPEzFvS;LAQKN2Tl!e`3jj(ICnhe^F$WpGD3Baca~=R zdMLqXpO)u&-sx_Tlcwd^38Ij*?$BVUpDnCfxh?c!wr-Te5TDFK~>-!4t5SR6erXTs3}ZlOqD zQkx-$C(;MU5*-avgqJ_ye-E`x0A`C5n>QZ8g>h_qB~P6+MF@Q8)YYf`h1-Yxy7N$P zPam5tT95Ng97?I$FNlw^p)j7+byrLq_9%9j7q}0!J+qZwNM)s}1M@ZJDvx($W-Zd& z-dCSmAA9jMC*~kLd_~;G#LX^_?}_nrG`j3>%!I*(6p>kr3e;r zUHXiVH=o?Cjm%&joG#YsQvd!-Y3tRT86h4p#lLHr9X%P^+HN}5o0~+GK3Vx=e!wQX zWS(7KoSLQJzx+gKB!j%wm82r2#1zuatS7sele^9I_;11GK^>*iL+yP8 zihZ#%!LO^F$Rzf~;Y);mxhAu@3SHWwndV^}bL}0Oq8a-fDdYK&CYOgZxOxjcwp!gb z95c?&fuflSbO%QLcAm+7=h@b}5Z(p$^J&mmnoP4tU%M%x3(c+~4M_*0%%&eA5AWI{ z?1(o=*S}M|x_p1KR!E6;;pp_i9)t0uFs0#Sxa&=%h0)M_ooNJ#9FGyrZ68(I5pO5W z?^p%&RNkrg9Sakp>SAvZJaLz+0)=`aD!uQ?YVpsQ9`Ej5ZjL|3lBfq?%v9Y@&CE8} z!Sc5JtgI9~%+$KmH?l+M`59->wSW3ECPj4eK!lmI7oR<0#OsN!G%@x3_)@21=f~tL zF=u?joql?|fvt@Zt%Dmmp)9PF38`QCk$o8rzFW-y0yPm3Aj;TkGR#-MXKWYZ(1z0! zvHFA&wRb2BQ=m6xT2fQ2nxk!1ORp`3{;o0Luv={^x7~f5nwqMvtBZ+^754Jt12Dy} zL)XmAjKk>=li7IKU@(y}mD4^z`}uChVzEBz?#|7Fj{xLtZ0_vD#K(&o88<`nUQ_Yj*e$n3)WRmM`|xB zSgB@{%k5Tz>)l@dT%9!m78cebC~OKEm!e5#@OgIjtow7EJdR)bc}&R%g(g6@H*xsa z7M{I+a6g^2+1eZK_bkgz>j^Zf%S?Q%jQ;jhEZN-#>KoGK#PV1qEo*{g2VcC0u7hUR z%ABy25VhBkczdE}DIkbrHH2ZQi#k=FLzDF>Jn(8>^jH`e&5{C@&Th&gnXIPjZo6dK1kJ>VOL)DC z(`h7nVx!~!EhBWOv3yZhM`+PSNV@emNDYTMd#KFYijK2;9g1XzY0VHYWJEeSA*gSU z^Y|Lc+32-?4Ck2TH8rXDz*{mo#kw(pw}~E^%fb%tt&W#}Clwo;(7MC! z^O=18D5tg7hfW*OS?Rt0z(4WIHtoY{_{*Aa%FZFn<-x`4b`tjA#-J=Gr`-FnxZ@>9 zZbJzQ>w@_oZgg93E#Ocd4ih{qw-iz#Ny}*7Lk`K>k7A|nyFD!PdH6n<3Dq0Sk# zI_GCXoX4`uK!Lcxt+3?9|9bVD4%FN7!u^tpuvtrbj zu(NI1fn#R*`-CrD=aA)6N!csj!I!9e)3~sQtk&BZWY`jJUP(jn zbOb&T8T+lFK4qUcr?>4!$3tFQ0=_k(%sgCdBqZ~y#Ct2{dF7MUZNB%;^q#k?a~{&uzg-q^b{V$5rQds{5ISc0@L`+nkBxt&9Fgg{d+qw$6dRB=2M34L zYt6_TN45PqYuNbs_~_JBNg&qO)Wm%zJWcl&1+%sSQ(RmeWU{sG^!?Of=w$|yGAJL_ zu#@ht((1t(k|nFI{uN}Z4aC#lSGj&f4&=05f79f2wB8#a(+G8#%2 zF*GFI+uJ+2ANueqAV%cFrzgQEAMA9`f40lzrRytRHthaReK9f)t1iPteb @jwv zm4ai$EEewRGW>Adc?U5()SX%~a1T(~vc;1A3Dqd@H)ZWMm;fv~cF^fF%Q zXIRKdt7zUWOL&?>;UXM&4=eUPb>7n1hEOXHbt~uWGvV!KjGEOawa=#~_4muxRd=%N zeVnd2+<$nV)LPHIGdAbN_Xq~;&E@#R-L*EnwHE{Qx2gENmvT4BBiD=69m8d{5xBMQ zY0L2CRJ<0_G*pu!iy1@ZG$=*sq8}`H`CD@xCNC+ombttxuUA%COV{%GA3lgMVO(Us zfqJdwwntiPG50skdAaWg3<7XCCM^ABpmMNuI!~XB*S{EqU6kE7z5;@osINe)p8Sd6 zK9X#BTU`V53RDa>1H#Fbt|WW{m9%2LWM%gRTG_7Jsd2PiUN<`-LcBI4-WkyC3c&|< z6suMRc_OO(LevRU z6AFYZ0`Gxu&s9-6YMc|)fF0;k_&XEys_?^qbBnSZ>7~bn$OpSf*de@0;-S_C zR!UrSO;YoupT3*0NJuJ%*!GqUXCJ%+Ta{op3SjcjHo&irobeZbyaGVeuXb zF4CBide>FS{g<|!AAgHCQJSx>%Ul$M>dV46Si@OZQEUoZlOOh!ZJ*f*w(@RaJZK_| zRr=ChU?M%J!#y_BY zA4w{WzoS;wQsJb^;59!aEyi{aJ~O3!$0o*f3)7c!Wke^P=}+6Xb&;J#{b-H#XD5E#}-~_wOLhnV2EG!G9~?QRUI< zY-LgAXyjCoEjouF(esVW^M=nNyErgF%38zVhp@}~;sO$v@t}e$MAH4Xr%C0@1ajNDd zL3t-gsD&UTm^_q3P{XE;P~y)4X+g4$d$i5$pk4lTewL2H4=(~A*^AVZpA?CMPSNuG zPv&p=L{K8J;h9#Eehi6M_5`aMwfiy9P2a%2^_x1^4<8aYi#{MqADs#Ie?)ZYNW`<^ zq2o4!CQzEu6IMiHD+=Rz&bkr^&(!bMi}yeW1MAcl4Taq!IL%JyjwwjAb&Ix{qs2IS z2rBJ*>jL+-Te3bn+NNLQCo0BAFjHP`1nPhN=qw;n@Bcr4zWl~W)khTk-(UF8|3pb| zKSa1_HfJ5=9lJq6Rj=b|d*eMy$H5UnN^- zdx?TwWPnOtSV+ZYG5hD~@qX{*M2t_H*ze1%jjY@kadF-4!6X{Z#@M}y{Eagg68%>- zbqOYxNKwf#CcM16W|Gt5eyG8rBSy@57l_6B)a7xJU8zyFXVQy%*nXa?mN4FM!oHQV zU;s2RSh$p{ebCMjm9%UwZb7;5i6vd^d7;$3NYwDXaU_T!Nhv|`_XVpwh6z#Ac;7<> z97qgJyavk_HGvhz&bv2IpcMu41BhAOxjEa2{v;SAW2Irj#OQRWg70}Dciem~{Sw~b z=+!BmL3TRr_7|>blvvkQzz;QJk?^lk2b&>PDRqKAEoI9MzU)l<4Dt~O=!13ORHXI0 zd6Q)MlMU7`?>n!lf{_n@PGwW~0O*GX{W`c0oTi+h__J1WI1E~JlEQ8`QN9FSMBdf! z{B$>oN`X$(^7p4e3;;nJ<(qXe*N%?bBx^YX7g5%N)uX+DZ0dfM3q#C4(i{U6QkhQuLy0AN7kUK zc(p+Q>iPsD*`%YIy1MIe!;XoW8Foq1=bK*LtSNGV7@thk^>2cT!MOaQo1Kgop?mj^ zjj+5msJqcj5J3+&6UN<=z|SbzNBX!+(2RxH4{wOaQNTaU0(KpcgnyfzzK@WwXv75X z52o(((_<=!EtC}58W>d}a?mg!+XY2MIB_dt<39wlE*(w0+L{r;ja2qP4h){0%F>oR zql7?NH)&Lm`rTZ}5wBX}D4vN=IB?w&J6wq6Jouw#l-}fYB(=W9dz9-73;pg zJU!-^s)+Jt6t;Qi2wNdl+G2zJ*X-fH^|znGB1P6v$GTywpPr(iJJZuc54gq02n$Xa zXKNZiQNqm`llgrKQhED;=ZipmBC7Wmg(120b(rbsPGn}_@ff*p?Cajos&Hn-xe%C0z!nx2?d@Z`PKlFzz=|UmA7vdKUSTxBQ}@}q!;#gna<%nC4VZxPf-=`8u6Cd4 z_0BX)Wp}@v!M7ht`>rtfQ~s4Jvp^KU@4`^R9;40nHkk&(Ve(>g`OyxO*CzV%OGXfB z$xOl}vXcph5NEvYBb$SVwy8@Bg+`26z`0xRQr$VQE{%;4u#{F zx)Kv?LqHUan%s*HLz60<|7I0sm_gDCg-qr{$bf%gq0osq(S|+IdT{B1Npe8x0sL?1 zj)||rc$iu{VG-mV3+ckKOQ#of)0Lrh8^MDE!f{Kdt-t0hV5{J}j<9MNF+$O3kY&-c z#!Lh;j`)M!(5+L@#!iBEa{CMS0vXV8YlJmQ zs&wUN3x$td3LCHzov41VZ=Q`rkU?5mM&ZeBYR^~l=r3IJI+&)3Bc{)9&O%h|Q(sd3 zQ{5LHQ`;^WU>SE#*E()ITK#e`5Mj?US~lkQ9M+w1>eSU{9Dz`kDf>32Pgq|516zu1 zle~gr*TLn7a8Yn0b8>Pp=YvFu-UGcVId2UlfxGYouu=gF|IQy+c#4o83rF2+Hkk40 zd8QzAB(w6XxHzLjmYkv@EG%q5sV7KCXR7W%3m1Jc&NGgw(&Tgr1Le*lGnel#=_o1T zo0iglA_sM>=}Z@^f^&T-e|!s?gLVOvjTRlBX9@@lSPC?%!iwzCH+cYoWjBvrEK(g{ z!#t#OBX{LQOb#8qn8<-0qqXFih#!yx<U`EXXJ+`$q5ZtPcS82K9>_ zxR9jiagl+4dS%m~0jj774bY*!MGY1PacC|)_(BU%nloo^Ed_-kQ)aByv>eKyoG~dW zsa;xMc_oL}!;S1hY7jh1+e&0%;j9Il23BZjF7{ZRuLQ-(Njk?rj4NSNt9TU*luMN@kBx7nx8U0D^CFSfQ7G|^#D zfZXsgX)wOH_AFw%yWnOhngo>^_{wf;VGfif5Xh|5md?o6YLgDoo7xlr@u+QX|t^m8uNZpSJ z_>V5vyY*2g;yhY^S1&&W2WfYdr80AQrMX{bMY&={CfJJB@pepv5-7?j%weS^oZZ9~ zG~=dg5$(61x{>p(;)w}-3!_DVL5e9QE5=7U`G$oDjh422nwawPkQhgIFX*mbo)$`Q z1A(yyTP7BdjPAH1dUfc(C4%(jF3=~jZb;lfn{mu`Q}2)-@x+Tr;WKK$SgbtXj7&0lnr-$2sT%dK#)tFodzip%f_jQ6Lf`EUx)^dR%WoR&!Gp4^^ zq61qhW26VrOjAoskpE(PIi(yE7iXxVb#--B>2_%b=zU*bpN8k<6bUIQR=k$GJGc2v zDJgciyqsM7VuL+WlOP!+w8Hb=2^jb>mT-|!2qSneI{N9b!~adITaz9p7mN`Wa&TBC zi6+d|-)v1cQidP3=Uzs|#&&`V;Yclln-_Z%hf7Tuii+YgGR9+BA^}c!FX5oeE$)m- zk_rk647TgS9s8ddFmnLjn4VSw%|p(&hlshk7s`zf4i5AeYAr#W0hrzi+FlKrmNGIj zC@+=e6cs`9kt9a_09}NE!a}=;8(VORuM3Kcf&J(#uM8C?YCa$4zPP&5V@qkc82<{+ z&R(5S34GO$cEaDke}kLaGwL=oHr9T;)C73?2NnJT4N3g{CHphAu!YOC50<);I_zw; zpFt%%pzO0;NQ4rLMaX{p=9^~l7_OR{8eAS1R6q$-Rk7mY;sSCpfX?M{!;mkX z2J%>$&8KNVnls{&_X()G@tN@Z!qIl+03Gv%L{~%v^0Mm=-YYQG5}A!7boI$KXH3n^ z^uU#TPJcguHz-+vrHmmw{3= z)jyv5Ra{z{jRwV&k06E3lITP5&$P55pjjXuQ13TNgM@9cULmHa2$ayAS`F%{A$uB8EYFd&83LdC<3*LQ1*q3*IX=&+$ir9N#ln8iN)p=ZE$gvvL z=kPfEOA*uEdPhc#|N6lPz_z22OTY4MVg`N%QHtKnc;nBXS8OTaAtCRC$hm>vvcP{R z&{R@_04v9Ckwpaw9h;efLp;`@prFusxH$`ObOJSSXPcZ)WZeMrLWe-Gr{#Fkh*Qv*8nVolhhRwv4jK1XZm!6W z{T|^C!3@p}s{zc8@t^{_cY1+=>xRJQWyVOJHEpm;5wo(!jgQL zh=>3rg?a5IOr$~s0iJ~}A@pSw2a4L-E2CnQ8%GIIXm#P?;eS?<*B>4p+O09LV$%T- zu&M)&wOp+vHa;G^Q3W&7r?C{JMq4yIjYWzW{hj#dM~H z145ELMgfO{pHo$Z7TP6fWmTGK34+71+#q!eQgU)pnJJkpQru-_yTA{BEz8W*-EA4^ z?PX|gvpHRD&%A2-_3O{*1)%7G$avAexcUJdxIqXo_Z3rR{1>!P4udHohq&bGczn_W z7bwbkeT{bJ(J{gH?{@g{=w<~cZuvegEo#qm^c*`X18iq%u|KDMzt=~S`o3bqU>EAD zf&6WFgm!Q-^v?9zO%-Y7Fjms?X)`ldiSW*rdkcnWenNrqJ;^xp-Cb}f)izak_oBV` zeN2##sDy*}P0YSGITb(u??iKl47xGbq}`|T&c#S(GHM+i?VCl;pTf-q84CB_6_-nV z!EA(4_L3O}I0?N^KQ4UzbvJ1Hp#Cr34rqCpDDb}jgSrRFi9-PQ{V#6+^4tHD%>O?` z0H9v{-?D%oW}HEk@c-C?njK9O{Xe(pDMSN=|36L`Ch`CIMHxiVpkfG4usCSK{mBt2 z7i)agRga;7^a}73pU>ZqkF7e!-}wvrb*yFX9G;x8v$V9f0t`1R>(!&-PAEVLv_b{7vIBO(IOD!N%RvBy$7P?`~u47&!{$>uKP*V8epIgg+mSeiB zOSH-`q$WgT)8x*m)0a>0_!t!2zE6&LS6z&fCocV`-! z_(1c)b$zdE1&fg>@|9!($JA>gvIJezIaY#HH0bQH0^Pl?1w&Av%U=+j2q!t=470#= zUbcmmiL6UMy|54w5fO8mfUj=3TaQs_A?C)#=tdi7F8vL#jFl6Pco`#N+S=NJ^YlQ? z0Qc76aP#~&pr7tT@aW1)B*66+VZ6;JK;(Ec7m1tj(LxP1e?Bx1^IKhs+S^bF{#oh8 zXL~u`X{8FxAh$>1)ffe$O%& zCVKC5G~ZRB--CFha0@6@MDPAok!?d5&S^J}`1ka5{B+MNRUtSja7v2a3=Z2LS645r zh3`wKXm<1C)dC2=Tq&X!8Y#0r>ha&gO)?H%o#AN!kkkA>lyB7cP$9}rC8u|GB^pRJfNk^hYC;?09&((x$W^u*qK8iIg6qoK+615l z45Ni0b-Gsb5vVV`1rP%OIxDj8Bej9=-3DAVAP>X3rvpS+uvsuv<9BAD{Ay`gI(1|B zc+ZWWQ9=fDcH9|Jdf=kBB)R8}r^skE-ccM_VeV_VvbZ={RMu*?`$m-k6W1(KCY3|r z+qalZS1j)`-*!T7r^8=`h5e&^iaakfYPwX@eAiDAxP@e4GjsEe+ypHq7JA~Sw}8Ng zi@v}vP#Hg<`+Ci=&UAuI2qQ#M5l_wK+O6d!wNu&DL8*KLbzM(V5`u*G|%^d*RB9)k*Ij`)TS6D4^2`#(=WbN)RfObp{ zRoSHnLMhU(knwrGI6Bs3?sZ{&1|Cek&Wg1fDae}Z_3rVp(c|4^!0@*htAG^wa#{xm z%+?ATEhpflvhwn}N4f9`IsgTJiM>td?GASamut2$yc9%Arv>aOoy$4}p9zzx9PV0# zUoNjldHP$Q&pMqE_{65kq5wYsO{WEbXllw)E?(2aST)PzgJD81@PW^=#w71Y$eQB# z@s)V};$;8sFP=_7iGEXzTV7uFt91{Sq#v`Cd6`lJbWwYxjYr}JyqJ8t(FHfSE1A^G zw7yljkwWLB)%aMRZYME^-hqOdep14n>|aO=Gwdtsw1FxS+VIHb+%D{K;}N=(q#C)d zgm!%l=iC0C{4CsSSh)6e2RkvDAb3o+fA93L54tf-=%$|pi59*K&u~jk7%=Zpd?KYOzksZVo49fe@9jX-DR|BCCv*FK)8R6hv#3g zP7=~lenI0GC|wG$6Ayl5=7|IcF20#NP@sVHF?zEF46Kvx+>sMdmom?&TYZNOn4(fs zEljk!!Rh-vfX(2$2}98fqib3tUqcu9^dev^xt-nM(sKi4T0Xt%G^<%Zb3q&$kfmUme6F3d2#7FTpE>BhUlDDi4z_B`8rvQ9Gs;?^ zC)U3&2pTb>CToHVbK`eFqTLjkfxWk4?2?|{6l6Cc_JK#r=lcFv$GfAqg0R^g&)g&Z z@5UKs^73hsF)akas%~MlB7f!pZ{L|5AXTq-9LvfqNs{$TRzefNTL~e21ftH1eBlQC zTxYXS)DdN~yP=Us>u~SHb<`IFlh;ym=z9o}A70+`FF>h64)lF`@&Y`d{&qVB=_u7| zp*EaWy%x1Y!nxantCF6bhPDcuB;ONnujL+yR>dk6KLAVU7&kXH)xF%G0`3vSWp#sP zaMRt-dB1-LD{Hy>0}=<=RXs4ONZe{*VF3so#p-p)p96!DoWB&t!UoyZ>zsB(a!A1GgM5`f}VC{=a31<@VrG%`m6w(oNT$uCMVBf|T0su1CzM zP5$~Dw%Ad|r-B^pc&$9Ok(*Q4Vu`RHKcGtM*8%@Q1w7dG{-_Sfc9JP<;V&w{{%4X8 zx_}_!a@xHCqz@2tEN-V_T}yMoQ>~lNVEHm4V`I^kN;OP(r=s!j@BsgVm71WSr#D`1 zb`7ZEtxySrhC{a59sT5>X?(oc09ZTbRVj#kn=Vlo12JuYdJWLGr2?KL1lQ_)H<1A} z!OiXMN|!Uej`{SPsn5W?6QaLs;qIN@s$Ll-?AH^JLmxbLi0;-7ASE5C_i=M3UXDo_ z71V09Bu%~05S=y9$gmCbhU;pFR{0oyw$TF^IIhsS9S8|Y<#GLxKhX!oiT{kj6b9-w z2xJhnw`azW3Z`F6*rp<{rND41 zD+WH!ykb?Q`sX%A9>iYk#AU^r?PT+z!g;+jBO{~Jb~u7Rwz!A070}?O2JDIs#8E5T z0B4z)7wij%vJL`V&>p3u!BAEKo$NKLayhD*DR;lN8oRp(tax^DQMf{-v##yYatx^f zRM%X42{){sPE1bz`<1?hgw5MuHwegT>yOtfE3NRuvG?ksY>Yxkn`G&tx+kIEWzW^z z+4k%BzPC8QWLDbL?5L{@9QpAin>+;{g|`^RQAAs`kZ-v$>B zxZ>`awkO95H97NeZs+6G&AymZ9u&ZC{{SW^GSm3{bn+bG?+=rhN6820k{+1xcxVi2 zvSH%lb^;$9l9~Cb!o}X+p2Kk;J$Z=i>Ub$szb7J?txP6^FT6;(Owc_1W%x