Visions and Revisions - Original.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pickle\n",
    "import random\n",
    "import re\n",
    "\n",
    "import numpy\n",
    "from scipy import sparse\n",
    "import torch\n",
    "from transformers import BertTokenizer, BertForMaskedLM\n",
    "from transformers import DistilBertTokenizer, DistilBertForMaskedLM\n",
    "from transformers import RobertaTokenizer, RobertaForMaskedLM\n",
    "from transformers import GPT2Tokenizer, GPT2LMHeadModel\n",
    "\n",
    "#import logging\n",
    "#logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "tokenizer = None\n",
    "model = None\n",
    "loaded_model_name = None\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.corpus import cmudict\n",
    "dictionary = cmudict.dict()\n",
    "from g2p_en import G2p\n",
    "g2p = G2p()\n",
    "\n",
    "m = torch.nn.Softmax(dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "re_word = re.compile(r\"[a-zA-Z' ]+\")\n",
    "re_vowel = re.compile(r\"[aeiouy]\")\n",
    "def get_pron(tok):\n",
    "    tok = tokenizer.convert_tokens_to_string([tok])\n",
    "    if tok.startswith('##'):\n",
    "        tok = tok[2:]\n",
    "    if tok.startswith(' '):\n",
    "        tok = tok[1:]\n",
    "    if not re_word.match(tok):\n",
    "        # Punctuation\n",
    "        return []\n",
    "    if tok in dictionary:\n",
    "        pron = dictionary[tok][0]\n",
    "    else:\n",
    "        # Word not in CMU dict: guess using g2p_en\n",
    "        pron = g2p(tok)\n",
    "    return pron\n",
    "\n",
    "def get_meter(pron):\n",
    "    if pron == []:\n",
    "        return 'p'\n",
    "    meter = ''\n",
    "    for ph in pron:\n",
    "        # We ignore stress levels in favor of poetic scansion\n",
    "        if ph[-1].isdigit():\n",
    "            meter += 'u' if ph[-1] == '0' else '-'\n",
    "    return meter\n",
    "\n",
    "def get_rhyme(pron):\n",
    "    if pron == []:\n",
    "        return 'p'\n",
    "    rhyme = ''\n",
    "    for ph in reversed(pron):\n",
    "        rhyme = ph.replace('1', '').replace('2', '') + rhyme\n",
    "        if ph[-1].isdigit() and int(ph[-1]) > 0:\n",
    "            break\n",
    "    return rhyme\n",
    "\n",
    "def is_word_piece(model, tok):\n",
    "    if model.startswith('bert') or model.startswith('distilbert'):\n",
    "        return tok.startswith('##')\n",
    "    elif model.startswith('roberta') or model.startswith('gpt2'):\n",
    "        tok = tokenizer.convert_tokens_to_string([tok])\n",
    "        return re_word.match(tok) and not tok.startswith(' ')\n",
    "def join_word_pieces(toks):\n",
    "    word = ''\n",
    "    for tok in toks:\n",
    "        if tok.startswith('##'):\n",
    "            tok = tok[2:]\n",
    "        word += tok\n",
    "    return word\n",
    "\n",
    "def is_full_word(model_name, tok):\n",
    "    if model_name.startswith('bert') or model_name.startswith('distilbert'):\n",
    "        return re_word.match(tok) and not tok.startswith('##')\n",
    "    elif model_name.startswith('roberta') or model.startswith('gpt2'):\n",
    "        tok = tokenizer.convert_tokens_to_string([tok])\n",
    "        return re_word.match(tok) and tok.startswith(' ')\n",
    "\n",
    "def is_punctuation(tok):\n",
    "    tok = tokenizer.convert_tokens_to_string([tok])\n",
    "    return not re_word.match(tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_meter_dict(model_name):\n",
    "    print(\"Generating \" + model_name + '_meter_dict.pkl')\n",
    "    vocab = tokenizer.get_vocab()\n",
    "    vocab_size = len(vocab)\n",
    "    meter_dict = {}\n",
    "    word_pieces = torch.zeros([vocab_size])\n",
    "    for tok in vocab:\n",
    "        i = vocab[tok]\n",
    "        pron = get_pron(tok)\n",
    "        meter = get_meter(pron)\n",
    "        if meter not in meter_dict:\n",
    "            meter_dict[meter] = torch.zeros([vocab_size])\n",
    "        meter_dict[meter][i] = 1.0\n",
    "        if is_word_piece(model_name, tok):\n",
    "            word_pieces[i] = 1.0\n",
    "\n",
    "    pickle.dump((word_pieces, meter_dict),\n",
    "                open(model_name + '_meter_dict.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_rhyme_matrix(model_name):\n",
    "    print(\"Generating \" + model_name + '_rhyme_matrix.pkl')\n",
    "    vocab = tokenizer.get_vocab()\n",
    "    vocab_size = len(vocab)\n",
    "    rhyme_matrix = sparse.lil_matrix((vocab_size, vocab_size))\n",
    "    rhymable_words = torch.zeros([vocab_size])\n",
    "    rhyme_groups = {}\n",
    "    for tok in vocab:\n",
    "        i = vocab[tok]\n",
    "        pron = get_pron(tok)\n",
    "        rhyme = get_rhyme(pron)\n",
    "        if rhyme not in rhyme_groups:\n",
    "            rhyme_groups[rhyme] = []\n",
    "        rhyme_groups[rhyme].append((i, pron))\n",
    "    for rhyme in rhyme_groups:\n",
    "        if len(rhyme_groups[rhyme]) < 2:\n",
    "            continue\n",
    "        for i, pron1 in rhyme_groups[rhyme]:\n",
    "            rhymable = False\n",
    "            for j, pron2 in rhyme_groups[rhyme]:\n",
    "                # Words with identical pronunciations can't be used as rhymes\n",
    "                if pron1 != pron2:\n",
    "                    rhyme_matrix[i,j] = 1.0\n",
    "                    rhymable = True\n",
    "            if rhymable:\n",
    "                rhymable_words[i] = 1.0\n",
    "\n",
    "    rhyme_matrix = sparse.csc_matrix(rhyme_matrix)\n",
    "    pickle.dump((rhymable_words, rhyme_matrix), open(model_name + '_rhyme_matrix.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = None\n",
    "vocab_size = None\n",
    "meter_dict = {}\n",
    "word_pieces = None\n",
    "rhymable_words = None\n",
    "rhyme_matrix = None\n",
    "rhyme_tensors = {}\n",
    "def initialize_rhyme_and_meter(model, meter=False, rhymes=False):\n",
    "    global vocab, vocab_size, word_pieces, meter_dict, rhymable_words, rhyme_matrix\n",
    "    vocab = tokenizer.get_vocab()\n",
    "    vocab_size = len(vocab)\n",
    "    if meter:\n",
    "        try:\n",
    "            f = open(model + '_meter_dict.pkl', 'rb')\n",
    "        except FileNotFoundError:\n",
    "            create_meter_dict(model)\n",
    "            f = open(model + '_meter_dict.pkl', 'rb')\n",
    "        word_pieces, meter_dict = pickle.load(f)\n",
    "    else:\n",
    "        try:\n",
    "            f = open(model + '_meter_dict.pkl', 'rb')\n",
    "        except FileNotFoundError:\n",
    "            create_meter_dict(model)\n",
    "            f = open(model + '_meter_dict.pkl', 'rb')\n",
    "        word_pieces, _ = pickle.load(f)\n",
    "    if rhymes:\n",
    "        global rhyme_matrix\n",
    "        try:\n",
    "            f = open(model + '_rhyme_matrix.pkl', 'rb')\n",
    "        except FileNotFoundError:\n",
    "            create_rhyme_matrix(model)\n",
    "            f = open(model + '_rhyme_matrix.pkl', 'rb')\n",
    "        rhymable_words, rhyme_matrix = pickle.load(f)\n",
    "\n",
    "# Computes the model's predictions for a text with a given set of ranges\n",
    "# masked by single mask tokens.\n",
    "def compute_replacement_probs_for_masked_tokens(model, tokenized_text,\n",
    "                                                masked_indices):\n",
    "    n = len(masked_indices)\n",
    "    dim = [vocab_size] * n\n",
    "\n",
    "    tokenized_text = tokenized_text.copy()\n",
    "    shift = 0\n",
    "    for i1, i2 in masked_indices:\n",
    "        i1 -= shift\n",
    "        i2 -= shift\n",
    "        shift += (i2 - i1)\n",
    "        tokenized_text[i1:i2+1] = ['[MASK]']\n",
    "        \n",
    "    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
    "    tokens_tensor = torch.tensor([indexed_tokens])\n",
    "\n",
    "    with torch.no_grad():\n",
    "        outputs = model(tokens_tensor)\n",
    "        predictions = outputs[0]\n",
    "\n",
    "    replacement_probs = [None] * n\n",
    "    shift = 0\n",
    "    for k, (i1, i2) in enumerate(masked_indices):\n",
    "        i1 -= shift\n",
    "        i2 -= shift\n",
    "        shift += (i2 - i1)\n",
    "        replacement_probs[k] = [predictions[0, i1, :]]\n",
    "\n",
    "    return replacement_probs\n",
    "\n",
    "# Computes the model's predictions for a text with a given set of ranges\n",
    "# masked.\n",
    "def compute_probs_for_masked_tokens(model, tokenized_text, masked_indices):\n",
    "    n = len(masked_indices)\n",
    "    dim = [vocab_size] * n\n",
    "\n",
    "    multipart_words = False\n",
    "    tokenized_text = tokenized_text.copy()\n",
    "    for i1, i2 in masked_indices:\n",
    "        if i2 > i1:\n",
    "            multipart_words = True\n",
    "        tokenized_text[i1:i2+1] = ['[MASK]'] * (i2 - i1 + 1)\n",
    "\n",
    "    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
    "    tokens_tensor = torch.tensor([indexed_tokens])\n",
    "\n",
    "    with torch.no_grad():\n",
    "        outputs = model(tokens_tensor)\n",
    "        predictions = outputs[0]\n",
    "\n",
    "    probs = [None] * n\n",
    "    for k, (i1, i2) in enumerate(masked_indices):\n",
    "        word_probs = []\n",
    "        for i in range(i1, i2+1):\n",
    "            word_probs.append(predictions[0, i, :])\n",
    "        probs[k] = word_probs\n",
    "\n",
    "    if multipart_words:\n",
    "        # We need to compute a separate probability with only one mask token\n",
    "        # for each word, so that we can replace the multipart words with\n",
    "        # single-part words.\n",
    "        replacement_probs \\\n",
    "            = compute_replacement_probs_for_masked_tokens(model,\n",
    "                                                          tokenized_text,\n",
    "                                                          masked_indices)\n",
    "\n",
    "    else:\n",
    "        replacement_probs = probs\n",
    "\n",
    "    return probs, replacement_probs\n",
    "\n",
    "# Find words that could, if chosen for the masked indices, take us back to an\n",
    "# arrangement that has already been tried. Because we are compiling independent\n",
    "# lists of forbidden words for each index, this method can overcorrect.\n",
    "def find_forbidden_words(tokenized_text, masked_indices, forbidden_texts):\n",
    "    forbidden_words = [torch.ones((vocab_size,))\n",
    "                       for i in range(len(masked_indices))]\n",
    "    d = forbidden_texts\n",
    "    def f(d, start):\n",
    "        i = start\n",
    "        for tok in tokenized_text[start:]:\n",
    "            mask_num = None\n",
    "            mask_len = 0\n",
    "            for k, (i1, i2) in enumerate(masked_indices):\n",
    "                if i == i1:\n",
    "                    mask_num = k\n",
    "                    mask_len = i2 - i1 + 1\n",
    "                    break\n",
    "            if mask_num is not None:\n",
    "                reached_end = False\n",
    "                for option_tok in d.keys():\n",
    "                    if f(d[option_tok], i+mask_len):\n",
    "                        option_idx = tokenizer \\\n",
    "                                     .convert_tokens_to_ids([option_tok])[0]\n",
    "                        forbidden_words[mask_num][option_idx] = 0.0\n",
    "                        reached_end = True\n",
    "                return reached_end\n",
    "            else:\n",
    "                if tok in d:\n",
    "                    d = d[tok]\n",
    "                else:\n",
    "                    return False\n",
    "            i += 1\n",
    "        return True\n",
    "    if f(d, 0):\n",
    "        return forbidden_words\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "# Function to adjust the output of the model based on various options.\n",
    "def adjust_probs(model, probs, tokenized_text, start, end, masked_indices,\n",
    "                 modifier=None, match_meter=None, forbidden_texts=None,\n",
    "                 random_factor=False, discouraged_words=None,\n",
    "                 rhyme_with=None, rhymable_only=False, rhymable_with_meters=False,\n",
    "                 allow_punctuation=None, no_word_pieces=False,\n",
    "                 strong_topic_bias=False, topicless_probs=None):\n",
    "        \n",
    "    if forbidden_texts is not None:\n",
    "        forbidden_words = find_forbidden_words(tokenized_text,\n",
    "                                               masked_indices,\n",
    "                                               forbidden_texts)\n",
    "    else:\n",
    "        forbidden_words = None\n",
    "\n",
    "    adj_probs = [[u.clone() for u in t] for t in probs]\n",
    "    for k in range(len(adj_probs)):\n",
    "        for j in range(len(adj_probs[k])):\n",
    "            if random_factor:\n",
    "                noise = torch.randn_like(adj_probs[k][j])\n",
    "                noise = noise * random_factor + 1.0\n",
    "                adj_probs[k][j] *= noise\n",
    "\n",
    "            adj_probs[k][j] = m(adj_probs[k][j])\n",
    "\n",
    "            # Do not produce word pieces. There is no way to keep the model\n",
    "            # behaving reliably if we allow it to produce words that are not\n",
    "            # actually in its vocabulary.\n",
    "            if no_word_pieces:\n",
    "                adj_probs[k][j] *= (1.0 - word_pieces)\n",
    "                \n",
    "            if rhymable_only:\n",
    "                adj_probs[k][j] *= rhymable_words\n",
    "            if rhymable_with_meters:\n",
    "                for rhyme_meter in rhymable_with_meters:\n",
    "                    test_meter = get_meter(get_pron(rhyme_meter))\n",
    "                    meter_tensor = meter_dict[test_meter]\n",
    "                    meter_matrix = sparse.dia_matrix((meter_tensor, [0]),\n",
    "                                                     shape=(vocab_size, vocab_size))\n",
    "                    # Take the dot product of meter and rhyme\n",
    "                    mat = meter_matrix.dot(rhyme_matrix)\n",
    "                    vec = torch.from_numpy(mat.sum(0)).squeeze().to(dtype=bool)\n",
    "                    adj_probs[k][j] *= vec\n",
    "\n",
    "            if forbidden_words is not None:\n",
    "                adj_probs[k][j] *= forbidden_words[k]\n",
    "\n",
    "            if allow_punctuation is False:\n",
    "                adj_probs[k][j] *= (1.0 - meter_dict['p'])\n",
    "\n",
    "            if match_meter is not None:\n",
    "                test_meter = get_meter(get_pron(match_meter[k]))\n",
    "                meter_tensor = meter_dict[test_meter]\n",
    "                if allow_punctuation is True:\n",
    "                    adj_probs[k][j] *= (meter_tensor + meter_dict['p'])\n",
    "                else:\n",
    "                    adj_probs[k][j] *= meter_tensor\n",
    "\n",
    "            if modifier is not None:\n",
    "                adj_probs[k][j] *= modifier\n",
    "            if discouraged_words is not None:\n",
    "                adj_probs[k][j] *= discouraged_words\n",
    "\n",
    "            if rhyme_with is not None:\n",
    "                for rhyme_word in rhyme_with:\n",
    "                    rhyme_idx = tokenizer.convert_tokens_to_ids([rhyme_word])[0]\n",
    "                    rhyme_tensor = rhyme_matrix[rhyme_idx, :].todense()\n",
    "                    rhyme_tensor = torch.from_numpy(rhyme_tensor)\n",
    "                    rhyme_tensor = rhyme_tensor.squeeze()\n",
    "                    adj_probs[k][j] *= rhyme_tensor\n",
    "\n",
    "            if strong_topic_bias:\n",
    "                bias_factor = (m(probs[k][j]) / m(topicless_probs[k][j])) ** strong_topic_bias\n",
    "                adj_probs[k][j] *= bias_factor\n",
    "\n",
    "    return adj_probs\n",
    "\n",
    "# Compute a score indicating how well the model's predictions improve the\n",
    "# probability for certain words. If multiple words are chosen, it is\n",
    "# assumed that they are supposed to rhyme.\n",
    "def compute_score_for_tokens(probs1, probs2, tokenized_text,\n",
    "                             indices, relative):\n",
    "    n = len(indices)\n",
    "    dim = [vocab_size] * n\n",
    "    \n",
    "    existing_token_ids = [None] * n\n",
    "    for k, (i1, i2) in enumerate(indices):\n",
    "        existing_token_ids[k] = []\n",
    "        for i in range(i1, i2+1):\n",
    "            token = tokenized_text[i]\n",
    "            index = tokenizer.convert_tokens_to_ids([token])[0]\n",
    "            existing_token_ids[k].append(index)\n",
    "            \n",
    "    existing_words_prob = 1.0\n",
    "    if probs1:\n",
    "        for k in range(n):\n",
    "            existing_word_prob = 0.0\n",
    "            for i, tok_id in enumerate(existing_token_ids[k]):\n",
    "                prob_tensor = probs1[k][i]\n",
    "                existing_word_prob += prob_tensor[tok_id]\n",
    "            existing_word_prob /= len(existing_token_ids[k])\n",
    "            existing_words_prob *= existing_word_prob\n",
    "        \n",
    "    if n == 1:\n",
    "        prob_tensor = probs2[0][0]\n",
    "        prediction_prob = torch.max(prob_tensor)\n",
    "        idx = prob_tensor.argmax().item()\n",
    "        predicted_token_ids = [idx]\n",
    "\n",
    "    elif n == 2:\n",
    "        # We compute scores for possible rhyme pairs using sparse matrix\n",
    "        # arithmetic. We use scipy instead of torch because torch's sparse\n",
    "        # tensors do not support the .max() function.\n",
    "        left_mat = sparse.dia_matrix((probs2[0][0], [0]), shape=dim)\n",
    "        mat = left_mat.dot(rhyme_matrix)\n",
    "        right_mat = sparse.dia_matrix((probs2[1][0], [0]), shape=dim)\n",
    "        mat = mat.dot(right_mat)\n",
    "        prediction_prob = mat.max()\n",
    "        idx = mat.argmax()\n",
    "        predicted_token_ids = list(numpy.unravel_index(idx, dim))\n",
    "    \n",
    "    if probs1:\n",
    "        if relative:\n",
    "            score = existing_words_prob / prediction_prob\n",
    "        else:\n",
    "            score = existing_words_prob\n",
    "    else:\n",
    "        score = prediction_prob\n",
    "        \n",
    "    predicted_tokens = [None] * n\n",
    "    for i in range(n):\n",
    "        predicted_tokens[i] \\\n",
    "            = tokenizer.convert_ids_to_tokens([predicted_token_ids[i]])[0]\n",
    "\n",
    "    return predicted_tokens, score\n",
    "\n",
    "# Tokenize a text and figure out (as best we can) its rhyme scheme.\n",
    "def process_text(model, text, start, end, match_rhyme, strip_punctuation=False):\n",
    "    lines = text.split('\\n')\n",
    "    \n",
    "    tok_index = start\n",
    "    toks = []\n",
    "    rhyme_types = {}\n",
    "    multipart_words = {}\n",
    "    fixed = False\n",
    "    fixed_toks = set()\n",
    "    line_ends = set()\n",
    "    for line in lines:\n",
    "        if model.startswith('roberta') or model.startswith('gpt2'):\n",
    "            line = ' ' + line\n",
    "\n",
    "        # Check for the special '[]' characters that indicate fixed text.\n",
    "        line_new = ''\n",
    "        shift = 0\n",
    "        fixed_chars = set()\n",
    "        for i, ch in enumerate(line):\n",
    "            if (model.startswith('bert') or model.startswith('distilbert')) and ch == ' ':\n",
    "                # BERT tokenizer strips spaces, so we must account for that.\n",
    "                shift += 1\n",
    "            if ch == '[':\n",
    "                fixed = True\n",
    "                shift += 1\n",
    "            elif ch == ']':\n",
    "                fixed = False\n",
    "                shift += 1\n",
    "            else:\n",
    "                line_new += ch\n",
    "                if fixed:\n",
    "                    fixed_chars.add(i - shift)\n",
    "        \n",
    "        line_toks = tokenizer.tokenize(line_new)\n",
    "        line_fixed_toks = set()\n",
    "        i = 0\n",
    "        for j, tok in enumerate(line_toks):\n",
    "            tok = tokenizer.convert_tokens_to_string([tok])\n",
    "            if tok.startswith('##'):\n",
    "                tok = tok[2:]\n",
    "            nchars = len(tok)\n",
    "            for k in range(nchars):\n",
    "                if i+k in fixed_chars:\n",
    "                    line_fixed_toks.add(j + tok_index)\n",
    "                    break\n",
    "            i += nchars\n",
    "        \n",
    "        if strip_punctuation:\n",
    "            stripped_line_toks = []\n",
    "            stripped_fixed_toks = set()\n",
    "            shift = 0\n",
    "            for j, tok in enumerate(line_toks):\n",
    "                if is_punctuation(tok):\n",
    "                    shift += 1\n",
    "                else:\n",
    "                    stripped_line_toks.append(tok)\n",
    "                    if j + tok_index in line_fixed_toks:\n",
    "                        stripped_fixed_toks.add(j + tok_index - shift)\n",
    "            line_toks = stripped_line_toks\n",
    "            line_fixed_toks = stripped_fixed_toks\n",
    "        \n",
    "        toks += line_toks\n",
    "        fixed_toks.update(line_fixed_toks)\n",
    "\n",
    "        # Check for multipart words.\n",
    "        word_bounds = []\n",
    "        for i, tok in enumerate(line_toks):\n",
    "            if is_word_piece(model, tok) or tok in (\"'\", \"s\", \"st\", \"d\",\n",
    "                                                    \"ve\", \"re\", \"nt\", \"ll\",\n",
    "                                                    \"t\", \"m\"):\n",
    "                if not word_bounds:\n",
    "                    word_bounds.append([i, i])\n",
    "                else:\n",
    "                    word_bounds[-1][1] = i\n",
    "            else:\n",
    "                word_bounds.append([i, i])\n",
    "        for i1, i2 in word_bounds:\n",
    "            if i1 == i2:\n",
    "                continue\n",
    "            for i in range(i1, i2+1):\n",
    "                multipart_words[i + tok_index] = (i1 + tok_index,\n",
    "                                                  i2 + tok_index)\n",
    "\n",
    "        if match_rhyme:\n",
    "            rhyme_type = None\n",
    "            # Only check rhyme for the last non-punctuation word of a line.\n",
    "            word = ''\n",
    "            i = len(line_toks) - 1\n",
    "            while i >= 0:\n",
    "                if i + tok_index in multipart_words:\n",
    "                    i1, i2 = multipart_words[i + tok_index]\n",
    "                    word = join_word_pieces(line_toks[i1-tok_index:i2-tok_index+1])\n",
    "                    i = multipart_words[i + tok_index][0] - tok_index\n",
    "                else:\n",
    "                    word = line_toks[i]\n",
    "\n",
    "                pron = get_pron(word)\n",
    "                if pron != []:\n",
    "                    rhyme_type = get_rhyme(pron)\n",
    "                    if rhyme_type is not None:\n",
    "                        if not rhyme_type in rhyme_types:\n",
    "                            rhyme_types[rhyme_type] = []\n",
    "                        rhyme_types[rhyme_type].append(tok_index + i)\n",
    "                        break\n",
    "                \n",
    "                i -= 1\n",
    "            \n",
    "        tok_index += len(line_toks)\n",
    "        line_ends.add(tok_index)\n",
    "\n",
    "    if match_rhyme:\n",
    "        rhyme_groups = {}\n",
    "        for rhyme in rhyme_types:\n",
    "            tok_list = rhyme_types[rhyme]\n",
    "            # Rhyme groups of more than two not currently supported, so we\n",
    "            # split the groups up into pairs\n",
    "            for i in range(0, len(tok_list), 2):\n",
    "                group = tok_list[i:i+2]\n",
    "                for index in group:\n",
    "                    rhyme_groups[index] = group\n",
    "\n",
    "        return toks, fixed_toks, multipart_words, rhyme_groups, line_ends\n",
    "    \n",
    "    else:\n",
    "        return toks, fixed_toks, multipart_words, {}, line_ends\n",
    "\n",
    "# Alters a text iteratively, word by word, using the model to pick\n",
    "# replacements.\n",
    "def depoeticize(text, max_iterations=100,\n",
    "                match_meter=False, match_rhyme=False, topic=None,\n",
    "                randomize=False, cooldown=0.01, modifier=None,\n",
    "                forbid_reversions=True, preserve_punctuation=False,\n",
    "                topic_prefix=\"The following poem is about\",\n",
    "                topic_postfix=\"The preceding poem was about\",\n",
    "                strong_topic_bias=False, stop_score=0.001,\n",
    "                discourage_repetition=False, stopwords=stopwords.words('english'),\n",
    "                model_name='bert-base-uncased',\n",
    "                sequential=False, verbose=True):\n",
    "    stopwords = set(stopwords)\n",
    "    \n",
    "    global tokenizer, model, loaded_model_name\n",
    "    if loaded_model_name != model_name:\n",
    "        if model_name.startswith('distilbert'):\n",
    "            tokenizer = DistilBertTokenizer.from_pretrained(model_name)\n",
    "            model = DistilBertForMaskedLM.from_pretrained(model_name)\n",
    "            bos_token = '[CLS]'\n",
    "            eos_token = '[SEP]'\n",
    "        if model_name.startswith('bert'):\n",
    "            tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "            model = BertForMaskedLM.from_pretrained(model_name)\n",
    "            bos_token = '[CLS]'\n",
    "            eos_token = '[SEP]'\n",
    "        if model_name.startswith('roberta'):\n",
    "            tokenizer = RobertaTokenizer.from_pretrained(model_name)\n",
    "            model = model = RobertaForMaskedLM.from_pretrained(model_name)\n",
    "            bos_token = tokenizer.bos_token\n",
    "            eos_token = tokenizer.eos_token\n",
    "        model.eval()\n",
    "\n",
    "    initialize_rhyme_and_meter(model_name, meter=match_meter,\n",
    "                               rhymes=match_rhyme)\n",
    "\n",
    "    if topic:\n",
    "        toks1 = tokenizer.tokenize(\"{0} {1} {2}: \"\n",
    "                                   .format(bos_token, topic_prefix, topic))\n",
    "        toks3 = tokenizer.tokenize(\" {0} {1}. {2}\"\n",
    "                                   .format(topic_postfix, topic, eos_token))\n",
    "    else:\n",
    "        toks1 = tokenizer.tokenize(\"{0} \".format(bos_token))\n",
    "        toks3 = tokenizer.tokenize(\" {0}\".format(eos_token))\n",
    "    start = len(toks1)\n",
    "    end = len(toks3)\n",
    "\n",
    "    toks2, fixed_toks, multipart_words, rhyme_groups, line_ends \\\n",
    "        = process_text(model_name, text, start, end, match_rhyme)\n",
    "    tokenized_text = toks1 + toks2 + toks3\n",
    "    n = len(tokenized_text)\n",
    "\n",
    "    forbidden_texts = {}\n",
    "\n",
    "    if sequential:\n",
    "        max_iterations = len(toks2)\n",
    "    for k in range(max_iterations):\n",
    "        last_score = 0.0\n",
    "        \n",
    "        if sequential and k >= len(tokenized_text) - start - end:\n",
    "            break\n",
    "            \n",
    "        # Discourage the selection of words already in the text, save for stopwords.\n",
    "        if discourage_repetition is not False:\n",
    "            discouraged_words = torch.ones((vocab_size,))\n",
    "            for i in range(start, n-end):\n",
    "                tok = tokenized_text[i]\n",
    "                if tok in stopwords:\n",
    "                    continue\n",
    "                idx = tokenizer.convert_tokens_to_ids([tok])[0]\n",
    "                discouraged_words[idx] = discourage_repetition\n",
    "        else:\n",
    "            discouraged_words = None\n",
    "        \n",
    "        # Compute the scores used to choose which word to change\n",
    "        outputs = [(None, None, float(\"inf\"))] * n\n",
    "        if sequential:\n",
    "            test_range = [start + k]\n",
    "        else:\n",
    "            test_range = range(start, n-end)\n",
    "        for i in test_range:\n",
    "            if preserve_punctuation:\n",
    "                if is_punctuation(tokenized_text[i]):\n",
    "                    continue\n",
    "            if i in fixed_toks:\n",
    "                continue\n",
    "            if i in multipart_words and i != multipart_words[i][0]:\n",
    "                # Only try the first part of a multipart word\n",
    "                continue\n",
    "                \n",
    "            if match_rhyme and i in rhyme_groups:\n",
    "                if i != rhyme_groups[i][0]:\n",
    "                    # Only try each rhyme group once\n",
    "                    continue\n",
    "                indices = rhyme_groups[i]\n",
    "            else:\n",
    "                indices = [i]\n",
    "                \n",
    "            indices = [multipart_words.get(idx, [idx, idx])\n",
    "                       for idx in indices]\n",
    "            if match_meter:\n",
    "                meter = [join_word_pieces(tokenized_text[i1:i2+1])\n",
    "                         for (i1, i2) in indices]\n",
    "            else:\n",
    "                meter = None\n",
    "                \n",
    "            if sequential:\n",
    "                probs1 = None\n",
    "                probs2 = compute_replacement_probs_for_masked_tokens(model,\n",
    "                                                                     tokenized_text,\n",
    "                                                                     indices)\n",
    "            else:\n",
    "                probs1, probs2 \\\n",
    "                    = compute_probs_for_masked_tokens(model,\n",
    "                                                      tokenized_text,\n",
    "                                                      indices)\n",
    "\n",
    "            # The strong topic bias feature compares the probs with and\n",
    "            # without the topic and biases the results in favor of words\n",
    "            # that are more probable with it.\n",
    "            if topic and strong_topic_bias:\n",
    "                topicless_indices = [(i1-start, i2-start)\n",
    "                                     for (i1, i2) in indices]\n",
    "                if sequential:\n",
    "                    topicless_probs1 = None\n",
    "                    topicless_probs2 \\\n",
    "                        = compute_replacement_probs_for_masked_tokens(model,\n",
    "                                                          tokenized_text[start:-end],\n",
    "                                                          topicless_indices)\n",
    "                else:\n",
    "                    topicless_probs1, topicless_probs2 \\\n",
    "                        = compute_probs_for_masked_tokens(model,\n",
    "                                                          tokenized_text[start:-end],\n",
    "                                                          topicless_indices)\n",
    "            else:\n",
    "                topicless_probs1 = None\n",
    "                topicless_probs2 = None\n",
    "                \n",
    "            if not sequential:\n",
    "                probs1 = adjust_probs(model, probs1, tokenized_text, start,\n",
    "                                      end, indices, modifier,\n",
    "                                      random_factor=randomize,\n",
    "                             strong_topic_bias=topic and strong_topic_bias,\n",
    "                             topicless_probs=strong_topic_bias and topicless_probs1)\n",
    "            probs2 = adjust_probs(model, probs2, tokenized_text, start,\n",
    "                                  end, indices, modifier,\n",
    "                                  meter, forbidden_texts,\n",
    "                                  discouraged_words=discouraged_words,\n",
    "                                  random_factor=randomize,\n",
    "                                  no_word_pieces=True,\n",
    "                         strong_topic_bias=topic and strong_topic_bias,\n",
    "                         topicless_probs=strong_topic_bias and topicless_probs2)\n",
    "            \n",
    "            predicted_tokens, score \\\n",
    "                = compute_score_for_tokens(probs1, probs2,\n",
    "                                           tokenized_text, indices,\n",
    "                                           relative=True)\n",
    "            outputs[i] = (indices, predicted_tokens, score)\n",
    "\n",
    "        # Choose a word to change\n",
    "        outputs.sort(key=lambda t: t[2])\n",
    "        chosen_indices = None\n",
    "        for (indices, predicted_tokens, score) in outputs:\n",
    "            if score >= stop_score:\n",
    "                break\n",
    "            if predicted_tokens is None:\n",
    "                continue\n",
    "            chosen_indices = indices\n",
    "            chosen_tokens = predicted_tokens\n",
    "            last_score = score\n",
    "            break\n",
    "\n",
    "        if chosen_indices is None:\n",
    "            if sequential:\n",
    "                continue\n",
    "            else:\n",
    "                break\n",
    "\n",
    "        # To prevent loops, we forbid the model from reverting to texts that it\n",
    "        # has already tried. The texts are stored in a trie (prefix tree) for\n",
    "        # efficient searchability.\n",
    "        if forbid_reversions:\n",
    "            d = forbidden_texts\n",
    "            for tok in tokenized_text:\n",
    "                if not tok in d:\n",
    "                    d[tok] = {}\n",
    "                d = d[tok]\n",
    "\n",
    "        # Make the actual revision and make note of what we've done.\n",
    "        change_made = False\n",
    "        new_token_indices = []\n",
    "        shift = 0\n",
    "        for j, (i1, i2) in enumerate(chosen_indices):\n",
    "            i1 -= shift\n",
    "            i2 -= shift\n",
    "            shift += (i2 - i1)\n",
    "            n -= (i2 - i1)\n",
    "            token = chosen_tokens[j]\n",
    "            if i2 > i1:\n",
    "                change_made = True\n",
    "                tokenized_text[i1:i2+1] = [token]\n",
    "                new_token_indices.append(i1)\n",
    "            elif tokenized_text[i1] != token:\n",
    "                change_made = True\n",
    "                tokenized_text[i1] = token\n",
    "                new_token_indices.append(i1)\n",
    "\n",
    "            for i in range(i1, i2+1):\n",
    "                if i in multipart_words:\n",
    "                    del multipart_words[i]\n",
    "            replacements = {}\n",
    "            for i in list(multipart_words.keys()):\n",
    "                if i > i2:\n",
    "                    j1, j2 = multipart_words[i]\n",
    "                    del multipart_words[i]\n",
    "                    replacements[i - (i2 - i1)] = (j1 - (i2 - i1),\n",
    "                                                   j2 - (i2 - i1))\n",
    "            for i in replacements:\n",
    "                multipart_words[i] = replacements[i]\n",
    "                    \n",
    "            replacements = {}\n",
    "            for i_old in list(rhyme_groups.keys()):\n",
    "               group = rhyme_groups[i_old].copy()\n",
    "               if i_old > i1:\n",
    "                   i_new = i_old - (i2 - i1)\n",
    "               else:\n",
    "                   i_new = i_old\n",
    "               group = [(idx - (i2 - i1) if idx > i1 else idx)\n",
    "                        for idx in group]\n",
    "               replacements[i_new] = group\n",
    "            rhyme_groups = replacements\n",
    "\n",
    "        if not change_made:\n",
    "            if sequential:\n",
    "                continue\n",
    "            else:\n",
    "                break\n",
    "\n",
    "        if verbose:\n",
    "            sample = tokenized_text[start:-end].copy()\n",
    "            for i in new_token_indices:\n",
    "                sample[i-start] = '<' + sample[i-start] + '>'\n",
    "            text = tokenizer.convert_tokens_to_string(sample)\n",
    "            print('-----------------------')\n",
    "            print('Iteration {0}, score = {1}'.format(k+1, last_score))\n",
    "            print(tokenizer.clean_up_tokenization(text))\n",
    "\n",
    "        if randomize and cooldown:\n",
    "            randomize *= (1.0 - cooldown)\n",
    "            \n",
    "    text = tokenizer.convert_tokens_to_string(tokenized_text[start:-end])\n",
    "    return tokenizer.clean_up_tokenization(text)\n",
    "\n",
    "# Generates a wholly new text by running a decoder model forward with the specified\n",
    "# constraints. This doesn't work very well.\n",
    "def parody(text, match_meter=False, match_rhyme=False, topic=None,\n",
    "           randomize=False, modifier=None, verbose=True,\n",
    "           topic_prefix=\"\", model='gpt2'):\n",
    "    model_name = model\n",
    "    \n",
    "    global tokenizer\n",
    "    tokenizer = GPT2Tokenizer.from_pretrained(model_name)\n",
    "    model = GPT2LMHeadModel.from_pretrained(model_name)\n",
    "    model.eval()\n",
    "    eos_token = tokenizer.eos_token\n",
    "    \n",
    "    initialize_rhyme_and_meter(model_name, meter=True, rhymes=match_rhyme)\n",
    "\n",
    "    if topic:\n",
    "        toks1 = tokenizer.tokenize(\"{0} {1} {2}. \"\n",
    "                                   .format(eos_token, topic_prefix, topic))\n",
    "    else:\n",
    "        toks1 = [eos_token]\n",
    "    start = len(toks1)\n",
    "\n",
    "    # We strip punctuation because, not being able to look ahead, the GPT-2\n",
    "    # model cannot reliably produce text that matches the punctuation of the\n",
    "    # original; the only way to get coherent output is to let the model decide\n",
    "    # on the punctuation.\n",
    "    toks2, fixed_toks, multipart_words, rhyme_groups, line_ends \\\n",
    "        = process_text(model_name, text, start, 0, match_rhyme,\n",
    "                       strip_punctuation=True)\n",
    "    \n",
    "    tokenized_text = toks1 + toks2\n",
    "    n = len(tokenized_text)\n",
    "    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
    "    \n",
    "    # As in Beckett's \"The Unnamable,\" we force the model to keep writing\n",
    "    # even when it wants to stop.\n",
    "    discouraged_words = torch.ones((vocab_size,))\n",
    "    eos_token_id = tokenizer.convert_tokens_to_ids([eos_token])[0]\n",
    "    discouraged_words[eos_token_id] = 0.0\n",
    "    newline_token_id = tokenizer.convert_tokens_to_ids(['\\n'])[0]\n",
    "    discouraged_words[newline_token_id] = 0.0\n",
    "\n",
    "    out_toks = indexed_tokens[:start]\n",
    "    i = start\n",
    "    just_added_punctuation = False\n",
    "    just_rhymed = False\n",
    "    while i < n:\n",
    "        if i in fixed_toks:\n",
    "            tok = indexed_tokens[i]\n",
    "            out_toks.append(tok)\n",
    "            tok = tokenizer.convert_ids_to_tokens([tok])[0]\n",
    "            just_added_punctuation = is_punctuation(tok)\n",
    "            i += 1\n",
    "            continue\n",
    "            \n",
    "        if match_rhyme and i in rhyme_groups:\n",
    "            rhyming = True\n",
    "            # We can't look ahead with this model, so the rhyming constraint\n",
    "            # only looks at words already chosen.\n",
    "            rhyme_words = [tokenizer.convert_ids_to_tokens([indexed_tokens[idx]])[0]\n",
    "                           for idx in rhyme_groups[i] if idx < i]\n",
    "            # ...but we do need to make sure to choose a word that can be rhymed\n",
    "            # with at least one word of the meter of later rhyming words.\n",
    "            rhyme_meters = [tokenizer.convert_ids_to_tokens([indexed_tokens[idx]])[0]\n",
    "                            for idx in rhyme_groups[i] if idx > i]\n",
    "        else:\n",
    "            rhyming = False\n",
    "            rhyme_words = None\n",
    "            rhyme_meters = None\n",
    "        \n",
    "        i1, i2 = multipart_words.get(i, [i, i])\n",
    "        if match_meter:\n",
    "            meter = [join_word_pieces(tokenized_text[i1:i2+1])]\n",
    "        else:\n",
    "            meter = None\n",
    "            \n",
    "        with torch.no_grad():\n",
    "            tokens_tensor = torch.tensor([out_toks])\n",
    "            outputs = model(tokens_tensor)\n",
    "            predictions = outputs[0]\n",
    "            \n",
    "        no_word_pieces = (i == start) or rhyme_words or just_added_punctuation or just_rhymed\n",
    "\n",
    "        probs = [[predictions[0, -1, :]]]\n",
    "        probs = adjust_probs(model, probs, None, 0, 0, None,\n",
    "                             modifier, meter,\n",
    "                             random_factor=randomize,\n",
    "                             discouraged_words=discouraged_words,\n",
    "                             allow_punctuation=not just_added_punctuation and not rhyme_words,\n",
    "                             no_word_pieces=no_word_pieces,\n",
    "                             rhyme_with=rhyme_words,\n",
    "                             rhymable_only=not match_meter and rhyming,\n",
    "                             rhymable_with_meters=match_meter and rhyme_meters)\n",
    "        \n",
    "        idx = probs[0][0].argmax().item()\n",
    "        if idx == eos_token_id:\n",
    "            break\n",
    "        \n",
    "        tok = tokenizer.convert_ids_to_tokens([idx])[0]\n",
    "        out_toks.append(idx)\n",
    "        \n",
    "        just_rhymed = not not rhyme_words\n",
    "\n",
    "        # Only proceed to the next input token if the output is a\n",
    "        # non-punctuation token.\n",
    "        if meter_dict['p'][idx] == 0.0:\n",
    "            if verbose and i in line_ends:\n",
    "                print('')\n",
    "            # Record the chosen token for rhyming purposes.\n",
    "            indexed_tokens[i] = idx\n",
    "            i += i2 - i1 + 1\n",
    "            just_added_punctuation = False\n",
    "        else:\n",
    "            # We don't allow multiple punctuation tokens in a row. This is\n",
    "            # because the model can potentially get stuck in a loop where it\n",
    "            # generates nothing but punctuation, in which case the process\n",
    "            # would never end.\n",
    "            just_added_punctuation = True\n",
    "            \n",
    "        if verbose:\n",
    "            string = tokenizer.convert_tokens_to_string([tok])\n",
    "            print(string, end='')\n",
    "\n",
    "    out = tokenizer.convert_ids_to_tokens(out_toks[start:])\n",
    "    text = tokenizer.convert_tokens_to_string(out)\n",
    "    return tokenizer.clean_up_tokenization(text)\n",
    "\n",
    "# Add modifier=metalness_modifier() to bias the results toward words that occur\n",
    "# frequently in heavy metal lyrics. First you will need to download the data set\n",
    "# available at https://github.com/ijmbarr/pythonic-metal.\n",
    "def metalness_modifier():\n",
    "    f = open('metalness.json', 'r')\n",
    "    metalness = json.load(f)\n",
    "    f.close()\n",
    "    vocab = tokenizer.get_vocab()\n",
    "    metalness_modifier = [0.0] * len(vocab)\n",
    "    for i, tok in enumerate(vocab):\n",
    "        if tok in metalness:\n",
    "            metalness_modifier[i] = metalness[tok]\n",
    "    return m(torch.tensor(metalness_modifier))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'may i invite thee on a sunny day? it is more sunny and less sunny : cold winds may blow the autumn leaves to may, and our day is not too late in may : always is where the light of nature comes, and never is thy fair complexion pale ; and different day is that mankind remains, or dies, on any other day, today ; but this eternal beauty shall none gain or take advantage of how much thou gives ; nor does thou as thou suffer in thy pain, but in unbroken time by which thou lives ; as long that breath can hear that breath can be, as thou knows it, and thou gives it to me.'"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = '''Shall I compare thee to a summer's day? \n",
    "Thou art more lovely and more temperate:\n",
    "Rough winds do shake the darling buds of May,\n",
    "And summer's lease hath all too short a date: \n",
    "Sometime too hot the eye of heaven shines,\n",
    "And often is his gold complexion dimmed; \n",
    "And every fair from fair sometime declines,\n",
    "By chance, or nature's changing course, untrimmed;\n",
    "But thy eternal summer shall not fade\n",
    "Nor lose possession of that fair thou ow'st;\n",
    "Nor shall Death brag thou wander'st in his shade,\n",
    "When in eternal lines to time thou grow'st; \n",
    "So long as men can breathe or eyes can see,\n",
    "So long lives this, and this gives life to thee.'''\n",
    "\n",
    "depoeticize(text,\n",
    "            model_name='bert-base-uncased',\n",
    "            max_iterations=50,\n",
    "            randomize=0.0,\n",
    "            cooldown=0.05,\n",
    "            stop_score=1.0,\n",
    "            match_meter=True,\n",
    "            match_rhyme=True,\n",
    "            preserve_punctuation=True,\n",
    "            topic=None,\n",
    "            strong_topic_bias=2.0,\n",
    "            sequential=True,\n",
    "            verbose=False,\n",
    "            modifier=None)#metalness_modifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Tyger Tyger, also known in the English as \"the lone wolf,\" created this cat, named \"T-Rex,\" by writing poetry'"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = '''[Tyger Tyger], burning bright, \n",
    "In the forests of the night; \n",
    "What immortal hand or eye, \n",
    "Could frame thy fearful symmetry?'''\n",
    "\n",
    "parody(text,\n",
    "       model='gpt2',\n",
    "       match_meter=True,\n",
    "       match_rhyme=True,\n",
    "       topic=\"a little, fluffy kitty cat\",\n",
    "       randomize=0.005,\n",
    "       modifier=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " I've received some great and funny mail\n",
      " from fans of Sonic the Hedge Labyrinth,\n",
      " so I thought I'm going to write nail-\n",
      " and pencil- on some of them. I'm synth-\n",
      " snikt, and I've never done\n",
      " a Sonic game, so I decided to \"\n",
      " in- character\" some of them. Although begun\n",
      " with my first Sonic, Sonic is into\n",
      " it now. Remember, Sonic is an old\n",
      " school, old- Nintendo game, so it's not\n",
      " like I've got all Sonic in my mold.\n",
      " I'm assuming that's what you're\n",
      " up to, though. I'll just say that I love\n",
      " it, so I'll... and I won't say of"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\n I\\'ve received some great and funny mail from fans of Sonic the Hedge Labyrinth, so I thought I\\'m going to write nail- and pencil- on some of them. I\\'m synth- snikt, and I\\'ve never done a Sonic game, so I decided to \" in- character\" some of them. Although begun with my first Sonic, Sonic is into it now. Remember, Sonic is an old school, old- Nintendo game, so it\\'s not like I\\'ve got all Sonic in my mold. I\\'m assuming that\\'s what you\\'re up to, though. I\\'ll just say that I love it, so I\\'ll... and I won\\'t say of'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This text is modified to ensure that the program can pick up on\n",
    "# all the rhymes—it makes no difference because the parody model\n",
    "# only looks at the rhyme and meter of the input.\n",
    "text = '''Shall I compare thee to a summer's day?\n",
    "Thou art more lovely and more fascinate:\n",
    "Rough winds do shake the darling buds of May,\n",
    "And summer's lease hath all too short a date: \n",
    "Sometime too hot the eye of heaven shines,\n",
    "And often is his gold complexion dimmed; \n",
    "And every fair from fair sometime declines,\n",
    "By chance, or nature's changing course, untrimmed;\n",
    "But thy eternal summer shall not fade\n",
    "Nor lose possession of that fair thou ow'st;\n",
    "Nor shall Death brag thou wander'st in his shade,\n",
    "When in eternal lines to time thou grow'st; \n",
    "So long as men can breathe or eyes can see,\n",
    "So long lives this, and this gives life to thee.'''\n",
    "\n",
    "parody(text,\n",
    "       model='gpt2',\n",
    "       match_meter=True,\n",
    "       match_rhyme=True,\n",
    "       randomize=0.005,\n",
    "       topic=\"Sonic the Hedgehog and his friend Tails\",\n",
    "       modifier=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------\n",
      "Iteration 1, score = 8.776559070611256e-07\n",
      "and did those feet in ancient time walk upon englands mountains green : and was the holy lamb of god, on <our> pleasant pastures seen! and did the countenance divine, shine forth upon our clouded hills? and was jerusalem builded here, among these dark satanic mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o clouds unfold! bring me my chariot of fire! i will not cease from mental fight, nor shall my sword sleep in my hand : till we have built jerusalem, in englands green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 2, score = 1.4563041759174666e-06\n",
      "and did those feet in ancient time walk upon <our> mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the countenance divine, shine forth upon our clouded hills? and was jerusalem builded here, among these dark satanic mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o clouds unfold! bring me my chariot of fire! i will not cease from mental fight, nor shall my sword sleep in my hand : till we have built jerusalem, in englands green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 3, score = 1.2741419368467177e-06\n",
      "and did those feet in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the countenance divine, shine forth upon our clouded hills? and was jerusalem builded here, among these dark satanic mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o clouds unfold! bring me my chariot of fire! i will not cease from mental fight, nor shall my sword sleep in my hand : till we have built jerusalem, in <our> green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 4, score = 3.8905913243070245e-05\n",
      "and did those feet in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the countenance divine, shine forth upon our clouded hills? and was jerusalem builded here, among these dark satanic mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o clouds <above>! bring me my chariot of fire! i will not cease from mental fight, nor shall my sword sleep in my hand : till we have built jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 5, score = 8.711371629033238e-05\n",
      "and did those <who> in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the countenance divine, shine forth upon our clouded hills? and was jerusalem builded here, among these dark satanic mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o clouds above! bring me my chariot of fire! i will not cease from mental fight, nor shall my sword sleep in my hand : till we have built jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 6, score = 8.093752694549039e-05\n",
      "and did those who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the countenance divine, shine forth upon our clouded hills? and was jerusalem builded here, among these dark satanic mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o clouds above! bring me my chariot of fire! i will not cease from <our> fight, nor shall my sword sleep in my hand : till we have built jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 7, score = 0.00010442874918226153\n",
      "and did those who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the countenance divine, shine forth upon our clouded hills? and was jerusalem builded here, among these dark satanic mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o <lord> above! bring me my chariot of fire! i will not cease from our fight, nor shall my sword sleep in my hand : till we have built jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 8, score = 0.00030197264277376235\n",
      "and did those who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the countenance divine, shine forth upon our <pleasant> hills? and was jerusalem builded here, among these dark satanic mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not cease from our fight, nor shall my sword sleep in my hand : till we have built jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 9, score = 0.0004590614989865571\n",
      "and did those who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the <trinity> divine, shine forth upon our pleasant hills? and was jerusalem builded here, among these dark satanic mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not cease from our fight, nor shall my sword sleep in my hand : till we have built jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 10, score = 0.0009720277739688754\n",
      "and did those who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant hills? and was jerusalem builded here, among these dark <abandoned> mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not cease from our fight, nor shall my sword sleep in my hand : till we have built jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 11, score = 0.0015268494607880712\n",
      "and did those who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant hills? and was jerusalem <hidden> here, among these dark abandoned mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not cease from our fight, nor shall my sword sleep in my hand : till we have built jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 12, score = 0.0018020764691755176\n",
      "and did those who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant hills? and was jerusalem hidden here, among these dark abandoned mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not <die> from our fight, nor shall my sword sleep in my hand : till we have built jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 13, score = 0.0022360284347087145\n",
      "and did those who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant hills? and was jerusalem hidden here, among these dark abandoned mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword sleep in my hand : till we have <reached> jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 14, score = 0.005293065682053566\n",
      "and did those who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant hills? and was jerusalem hidden here, among these dark abandoned mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword <be> in my hand : till we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 15, score = 0.006854649633169174\n",
      "and did <he> who in ancient time walk upon our mountains green : and was the holy lamb of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant hills? and was jerusalem hidden here, among these dark abandoned mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword be in my hand : till we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 16, score = 0.00705138873308897\n",
      "and did he who in ancient time walk upon our <pleasant> green : and was the holy lamb of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant hills? and was jerusalem hidden here, among these dark abandoned mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword be in my hand : till we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 17, score = 0.007936679758131504\n",
      "and did he who in ancient time walk upon our pleasant green : and was the holy <name> of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant hills? and was jerusalem hidden here, among these dark abandoned mills? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword be in my hand : till we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 18, score = 0.012525005266070366\n",
      "and did he who in ancient time walk upon our pleasant green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant <lands>? and was jerusalem hidden here, among these dark abandoned <sands>? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword be in my hand : till we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 19, score = 0.025439150631427765\n",
      "and did he who in <our> time walk upon our pleasant green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark abandoned sands? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword be in my hand : till we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 20, score = 0.018971562385559082\n",
      "and did he who in our time <shone> upon our pleasant green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark abandoned sands? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword be in my hand : till we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 21, score = 0.04953284189105034\n",
      "and did he who in our time shone upon our <pastures> green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark abandoned sands? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword be in my hand : till we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 22, score = 0.06036099046468735\n",
      "and did he who in our time shone upon our pastures green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark abandoned sands? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor shall my sword be in my hand : <for> we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 23, score = 0.05366590991616249\n",
      "and did he who in our time shone upon our pastures green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark abandoned sands? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor <will> my sword be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 24, score = 0.08315765112638474\n",
      "and did he who in our time shone upon our pastures green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark <eternal> sands? bring me my bow of burning gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my sword be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 25, score = 0.07688646763563156\n",
      "and did he who in our time shone upon our pastures green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark eternal sands? bring me my bow of <shining> gold : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my sword be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 26, score = 0.012502544559538364\n",
      "and did he who in our time shone upon our pastures green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark eternal sands? bring me my bow of shining <light> : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my sword be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 27, score = 0.12427441030740738\n",
      "and did he who in our time shone upon our pastures green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark eternal sands? bring me my <spear> of shining light : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my sword be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 28, score = 0.14537326991558075\n",
      "and did he who in our time shone upon our pastures green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark eternal sands? bring me my spear of shining light : bring me my arrows of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my <spear> be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 29, score = 0.1377590447664261\n",
      "and did he who in our time shone upon our pastures green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these dark eternal sands? bring me my spear of shining light : bring me my <weapon> of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 30, score = 0.13985151052474976\n",
      "and did he who in our time shone upon our pastures green : and was the holy name of god, on our pleasant pastures seen! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these <green> eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 31, score = 0.145905002951622\n",
      "and did he who in our time shone upon our pastures <light> : and was the holy name of god, on our pleasant pastures <bright>! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 32, score = 4.396847361931577e-05\n",
      "and did he who in our time shone upon our <shining> light : and was the holy name of god, on our pleasant pastures bright! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 33, score = 0.09347369521856308\n",
      "and did he who in our time shone upon our shining light : and <by> the holy name of god, on our pleasant pastures bright! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 34, score = 0.007772695738822222\n",
      "and did he who in our time shone upon our shining light : and by the holy name of god, <made> our pleasant pastures bright! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 35, score = 0.0888465866446495\n",
      "and did he who in our time shone upon our shining <land> : and by the holy name of god, made our pleasant pastures <stand>! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 36, score = 0.0010711896466091275\n",
      "and did he who in our time shone upon our <pleasant> land : and by the holy name of god, made our pleasant pastures stand! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 37, score = 0.008623515255749226\n",
      "and did he who in our time shone upon our pleasant land : and by the holy name of god, <let> our pleasant pastures stand! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 38, score = 0.03998076170682907\n",
      "and did he who in our time shone upon our pleasant land : and by the holy name of god, let our pleasant <country> stand! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 39, score = 0.10102830082178116\n",
      "and did he who in our time shone upon our pleasant land : and by the holy <grace> of god, let our pleasant country stand! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our green & pleasant land.\n",
      "-----------------------\n",
      "Iteration 40, score = 0.12009388208389282\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the trinity divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our <time> & pleasant land.\n",
      "-----------------------\n",
      "Iteration 41, score = 0.1853724867105484\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the <glorious> divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these green eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n",
      "-----------------------\n",
      "Iteration 42, score = 0.2745162844657898\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these <great> eternal sands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n",
      "-----------------------\n",
      "Iteration 43, score = 0.18242542445659637\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among these great eternal <hands>? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n",
      "-----------------------\n",
      "Iteration 44, score = 0.24834373593330383\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among <his> great eternal hands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n",
      "-----------------------\n",
      "Iteration 45, score = 0.2893253266811371\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant lands? and was jerusalem hidden here, among his great <almighty> hands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n",
      "-----------------------\n",
      "Iteration 46, score = 0.1610690802335739\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant lands? and was jerusalem <standing> here, among his great almighty hands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n",
      "-----------------------\n",
      "Iteration 47, score = 0.1984846591949463\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant lands? and was jerusalem standing here, <upon> his great almighty hands? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n",
      "-----------------------\n",
      "Iteration 48, score = 0.011117629706859589\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant <land>? and was jerusalem standing here, upon his great almighty <hand>? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my chariot of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n",
      "-----------------------\n",
      "Iteration 49, score = 0.4874098300933838\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant land? and was jerusalem standing here, upon his great almighty hand? bring me my spear of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my <instrument> of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n",
      "-----------------------\n",
      "Iteration 50, score = 0.46125081181526184\n",
      "and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant land? and was jerusalem standing here, upon his great almighty hand? bring me my <sword> of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my instrument of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'and did he who in our time shone upon our pleasant land : and by the holy grace of god, let our pleasant country stand! and did the glorious divine, shine forth upon our pleasant land? and was jerusalem standing here, upon his great almighty hand? bring me my sword of shining light : bring me my weapon of desire : bring me my spear : o lord above! bring me my instrument of fire! i will not die from our fight, nor will my spear be in my hand : for we have reached jerusalem, in our time & pleasant land.'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = '''And did those feet in ancient time\n",
    "Walk upon Englands mountains green:\n",
    "And was the holy Lamb of God,\n",
    "On Englands pleasant pastures seen!\n",
    " \n",
    "And did the Countenance Divine,\n",
    "Shine forth upon our clouded hills?\n",
    "And was Jerusalem builded here,\n",
    "Among these dark Satanic Mills?\n",
    " \n",
    "Bring me my Bow of burning gold:\n",
    "Bring me my arrows of desire:\n",
    "Bring me my Spear: O clouds unfold!\n",
    "Bring me my Chariot of fire!\n",
    " \n",
    "I will not cease from Mental Fight,\n",
    "Nor shall my sword sleep in my hand:\n",
    "Till we have built Jerusalem,\n",
    "In Englands green & pleasant Land.'''\n",
    "\n",
    "depoeticize(text,\n",
    "            model_name='bert-base-uncased',\n",
    "            max_iterations=50,\n",
    "            randomize=False,\n",
    "            cooldown=0.05,\n",
    "            stop_score=1.0,\n",
    "            match_meter=True,\n",
    "            match_rhyme=True,\n",
    "            preserve_punctuation=True,\n",
    "            topic=None,\n",
    "            strong_topic_bias=2.0,\n",
    "            modifier=None)#metalness_modifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------\n",
      "Iteration 1, score = 4.096679333542852e-07\n",
      "tyger tyger, burning bright, in the forests of the night ; what immortal hand or eye, could <break> thy fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 2, score = 1.5753139450680465e-06\n",
      "tyger tyger, burning bright, in the <middle> of the night ; what immortal hand or eye, could break thy fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 3, score = 5.9419497119961306e-05\n",
      "tyger tyger, <shining> bright, in the middle of the night ; what immortal hand or eye, could break thy fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 4, score = 0.0006441521691158414\n",
      "tyger tyger, shining bright, in the middle of the night ; what immortal hand or eye, could break thy <broken> symmetry?\n",
      "-----------------------\n",
      "Iteration 5, score = 0.0005108318291604519\n",
      "tyger tyger, shining bright, in the middle of the night ; what immortal hand or eye, could break <this> broken symmetry?\n",
      "-----------------------\n",
      "Iteration 6, score = 0.01689990423619747\n",
      "tyger tyger, shining bright, in the middle of the night ; what immortal <ear> or eye, could break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 7, score = 0.00964891817420721\n",
      "tyger tyger, shining bright, in the middle of the night ; what <enchanted> ear or eye, could break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 8, score = 0.07427377253770828\n",
      "tyger tyger, shining bright, in the middle of the night ; what enchanted ear or eye, <can> break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 9, score = 0.08845506608486176\n",
      "tyger tyger, shining <white>, in the middle of the night ; what enchanted ear or eye, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 10, score = 0.13678152859210968\n",
      "tyger tyger, <fluffy> white, in the middle of the night ; what enchanted ear or eye, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 11, score = 0.24476732313632965\n",
      "tyger tyger, fluffy <snake>, in the middle of the <cake> ; what enchanted ear or eye, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 12, score = 0.013579119928181171\n",
      "tyger tyger, <little> snake, in the middle of the cake ; what enchanted ear or eye, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 13, score = 0.0309988372027874\n",
      "tyger tyger, little snake, in the middle of the cake ; what enchanted <tongue> or eye, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 14, score = 0.11295196413993835\n",
      "tyger tyger, little snake, in the middle of <a> cake ; what enchanted tongue or eye, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 15, score = 0.018510691821575165\n",
      "tyger tyger, little snake, in the middle of a cake ; what <annoying> tongue or eye, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 16, score = 0.006332708057016134\n",
      "tyger tyger, little snake, in the middle of a cake ; <no> annoying tongue or eye, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 17, score = 0.013200025074183941\n",
      "tyger tyger, little snake, in the middle of a cake ; no <protruding> tongue or eye, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 18, score = 0.0012039612047374249\n",
      "tyger tyger, little snake, in the middle of a cake ; no protruding tongue or <claws>, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 19, score = 0.022402692586183548\n",
      "tyger tyger, little <cat>, in the middle of a <rat> ; no protruding tongue or claws, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 20, score = 0.002294139936566353\n",
      "tyger tyger, <kitty> cat, in the middle of a rat ; no protruding tongue or claws, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 21, score = 0.0026613755617290735\n",
      "tyger tyger, kitty cat, in the <posture> of a rat ; no protruding tongue or claws, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 22, score = 0.008783727884292603\n",
      "tyger tyger, kitty cat, in the posture of a rat ; no protruding <fangs> or claws, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 23, score = 0.1805858463048935\n",
      "tyger tyger, kitty cat, in the posture of a <bat> ; no protruding fangs or claws, can break this broken symmetry?\n",
      "-----------------------\n",
      "Iteration 24, score = 0.24231919646263123\n",
      "tyger tyger, kitty cat, in the posture of a bat ; no protruding fangs or claws, can break this <rigid> symmetry?\n",
      "-----------------------\n",
      "Iteration 25, score = 0.23465681076049805\n",
      "tyger tyger, kitty cat, in the posture of a bat ; no protruding fangs or claws, <what> break this rigid symmetry?\n",
      "-----------------------\n",
      "Iteration 26, score = 8.271900696854573e-06\n",
      "tyger tyger, kitty cat, in the posture of a bat ; no protruding fangs or claws, what <is> this rigid symmetry?\n",
      "-----------------------\n",
      "Iteration 27, score = 0.010092588141560555\n",
      "tyger tyger, kitty cat, in the posture of a bat ; no protruding fangs or claws, what is this rigid <animal>?\n",
      "-----------------------\n",
      "Iteration 28, score = 7.039792308205506e-06\n",
      "tyger tyger, kitty cat, in the posture of a bat ; no protruding fangs or claws, what is this <little> animal?\n",
      "-----------------------\n",
      "Iteration 29, score = 0.36962541937828064\n",
      "tyger tyger, kitty cat, in the posture of a bat ; no protruding fangs or claws, what is <that> little animal?\n",
      "-----------------------\n",
      "Iteration 30, score = 0.14315733313560486\n",
      "tyger tyger, kitty cat, in the posture of a bat ; no protruding fangs or claws, what is that <furry> animal?\n",
      "-----------------------\n",
      "Iteration 31, score = 0.08315431326627731\n",
      "tyger tyger, kitty cat, in the posture of a bat ; no protruding fangs or claws, what is <this> furry animal?\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'tyger tyger, kitty cat, in the posture of a bat ; no protruding fangs or claws, what is this furry animal?'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = '''[Tyger Tyger], burning bright, \n",
    "In the forests of the night; \n",
    "What immortal hand or eye, \n",
    "Could frame thy fearful symmetry?'''\n",
    "\n",
    "depoeticize(text,\n",
    "            model_name='bert-base-uncased',\n",
    "            max_iterations=50,\n",
    "            randomize=0.1,\n",
    "            cooldown=0.05,\n",
    "            stop_score=1.0,\n",
    "            match_meter=True,\n",
    "            match_rhyme=True,\n",
    "            preserve_punctuation=True,\n",
    "            topic=\"a little, fluffy kitty cat\",\n",
    "            strong_topic_bias=2.0,\n",
    "            modifier=None)#metalness_modifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------\n",
      "Iteration 1, score = 0.00013018323807045817\n",
      "o <god> thou art sick. the invisible worm, that flies in the night in the howling storm : has found out thy bed of crimson joy : and his dark secret love does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 2, score = 0.004045261535793543\n",
      "o god thou art sick. the invisible worm, that flies in the night in the howling storm : has found out thy <source> of crimson joy : and his dark secret love does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 3, score = 0.005103247240185738\n",
      "o god thou art <free>. the invisible worm, that flies in the night in the howling storm : has found out thy source of crimson joy : and his dark secret love does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 4, score = 0.010653567500412464\n",
      "o god thou art free. the invisible worm, that flies in the night in the howling storm : has found out thy source of <deepest> joy : and his dark secret love does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 5, score = 0.018104439601302147\n",
      "o god thou art free. the invisible <man>, that flies in the night in the howling storm : has found out thy source of deepest joy : and his dark secret love does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 6, score = 0.020744169130921364\n",
      "o god thou art free. the invisible man, <who> flies in the night in the howling storm : has found out thy source of deepest joy : and his dark secret love does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 7, score = 0.01984221860766411\n",
      "o god thou art free. the invisible man, who flies in the night in the howling storm : has found out thy source of deepest joy : and his dark secret love <makes> thy life destroy.\n",
      "-----------------------\n",
      "Iteration 8, score = 0.01053999736905098\n",
      "o god thou art free. the invisible man, who flies in the night in the howling storm : has found out thy source of deepest joy : and his dark secret love makes thy <heart> destroy.\n",
      "-----------------------\n",
      "Iteration 9, score = 0.018236761912703514\n",
      "o god thou art free. the invisible man, who flies in the night in the howling storm : has found out <his> source of deepest joy : and his dark secret love makes thy heart destroy.\n",
      "-----------------------\n",
      "Iteration 10, score = 0.006318132393062115\n",
      "o god thou art free. the invisible man, who flies in the night in the howling storm : has found out his source of deepest joy : and his dark secret love makes <his> heart destroy.\n",
      "-----------------------\n",
      "Iteration 11, score = 0.023261796683073044\n",
      "o god thou art free. the invisible man, who <walks> in the night in the howling storm : has found out his source of deepest joy : and his dark secret love makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 12, score = 0.02437368966639042\n",
      "o god thou art free. the invisible man, who walks in the night in the howling storm : has found out his source of deepest joy : and his dark secret <that> makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 13, score = 0.02021188475191593\n",
      "o god thou art free. the invisible man, who walks in the night in the howling storm : has found <both> his source of deepest joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 14, score = 0.12877380847930908\n",
      "o god thou art free. the invisible man, who walks in the night in the howling <wind> : has found both his source of deepest joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 15, score = 0.31052204966545105\n",
      "o god thou art free. the invisible man, who walks in the night in the <gentle> wind : has found both his source of deepest joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 16, score = 0.19000396132469177\n",
      "o god thou art free. the invisible man, who walks in the night in the gentle <light> : has found both his source of deepest joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 17, score = 0.13852651417255402\n",
      "o god thou art free. the invisible man, who walks in the night in the <golden> light : has found both his source of deepest joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 18, score = 0.41193529963493347\n",
      "o god thou art free. the invisible man, who walks in the night in the golden light : has found both his source of <inner> joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 19, score = 0.4820098578929901\n",
      "o god thou art free. the invisible man, who walks in the night in <a> golden light : has found both his source of inner joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 20, score = 0.3099149763584137\n",
      "o god thou art free. the invisible man, who walks in the night in a golden <cloak> : has found both his source of inner joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 21, score = 0.49906983971595764\n",
      "<by> god thou art free. the invisible man, who walks in the night in a golden cloak : has found both his source of inner joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 22, score = 0.1442061960697174\n",
      "by god thou art <blessed>. the invisible man, who walks in the night in a golden cloak : has found both his source of inner joy : and his dark secret that makes his heart destroy.\n",
      "-----------------------\n",
      "Iteration 23, score = 0.7762159705162048\n",
      "by god thou art blessed. the invisible man, who walks in the night in a golden cloak : has found both his source of inner joy : and his dark secret that makes his <soul> destroy.\n",
      "-----------------------\n",
      "Iteration 24, score = 0.3973541259765625\n",
      "by god thou art blessed. the invisible man, who walks in the night in a golden cloak : has found both his source of inner joy : and his dark <power> that makes his soul destroy.\n",
      "-----------------------\n",
      "Iteration 25, score = 0.15802240371704102\n",
      "by god thou art blessed. the invisible man, who walks in the night in a golden cloak : has found both his source of inner joy : and his <own> power that makes his soul destroy.\n",
      "-----------------------\n",
      "Iteration 26, score = 0.37217259407043457\n",
      "by god thou art blessed. the invisible man, who walks in the night in a golden cloak : has found both his source of inner <heat> : and his own power that makes his soul <complete>.\n",
      "-----------------------\n",
      "Iteration 27, score = 0.2677094638347626\n",
      "by god thou art blessed. the invisible man, who walks in the night in a golden cloak : has found both his source of inner heat : and his own power that makes his <life> complete.\n",
      "-----------------------\n",
      "Iteration 28, score = 0.8659397959709167\n",
      "by god thou art blessed. the invisible man, who walks in the night in a <hooded> cloak : has found both his source of inner heat : and his own power that makes his life complete.\n",
      "-----------------------\n",
      "Iteration 29, score = 0.8591411113739014\n",
      "by god thou art blessed. the invisible man, who walks in the night in a hooded cloak : has found both his source of <body> heat : and his own power that makes his life complete.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'by god thou art blessed. the invisible man, who walks in the night in a hooded cloak : has found both his source of body heat : and his own power that makes his life complete.'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = '''O Rose thou art sick. \n",
    "The invisible worm, \n",
    "That flies in the night \n",
    "In the howling storm: \n",
    "\n",
    "Has found out thy bed \n",
    "Of crimson joy: \n",
    "And his dark secret love \n",
    "Does thy life destroy.'''\n",
    "\n",
    "depoeticize(text,\n",
    "            model_name='bert-base-uncased',\n",
    "            max_iterations=50,\n",
    "            randomize=False,\n",
    "            cooldown=0.05,\n",
    "            stop_score=1.0,\n",
    "            match_meter=True,\n",
    "            match_rhyme=True,\n",
    "            preserve_punctuation=True,\n",
    "            topic=None,\n",
    "            strong_topic_bias=2.0,\n",
    "            modifier=None)#metalness_modifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------\n",
      "Iteration 1, score = 9.045047772815451e-05\n",
      " O Rose thou< his> sick. The invisible worm, That flies in the night In the howling storm: Has found out thy bed Of crimson joy: And his dark secret love Does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 2, score = 9.544840577291325e-05\n",
      " O< God> thou his sick. The invisible worm, That flies in the night In the howling storm: Has found out thy bed Of crimson joy: And his dark secret love Does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 3, score = 0.0001600701070856303\n",
      " O God thou his sick. The invisible< sweet>, That flies in the night In the howling storm: Has found out thy bed Of crimson joy: And his dark secret love Does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 4, score = 0.0010359423467889428\n",
      " O God thou his sick. The invisible sweet, That flies in the night In the< winter> storm: Has found out thy bed Of crimson joy: And his dark secret love Does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 5, score = 0.0019554041791707277\n",
      " O God thou his sick. The invisible sweet, That flies in the night In the winter storm: Has found out thy bed Of crimson joy: And his dark secret< that> Does thy life destroy.\n",
      "-----------------------\n",
      "Iteration 6, score = 0.002129178261384368\n",
      " O God thou his sick. The invisible sweet, That flies in the night In the winter storm: Has found out thy bed Of crimson< clay>: And his dark secret that Does thy life< away>.\n",
      "-----------------------\n",
      "Iteration 7, score = 0.0018448985647410154\n",
      " O God thou his sick. The invisible sweet, That flies in the night In the winter storm: Has found out thy bed Of crimson clay: And his dark secret that< steals> thy life away.\n",
      "-----------------------\n",
      "Iteration 8, score = 0.0016542761586606503\n",
      " O God thou his sick. The invisible< blood>, That flies in the night In the winter storm: Has found out thy bed Of crimson clay: And his dark secret that steals thy life away.\n",
      "-----------------------\n",
      "Iteration 9, score = 0.0016149417497217655\n",
      " O God< his> his sick. The invisible blood, That flies in the night In the winter storm: Has found out thy bed Of crimson clay: And his dark secret that steals thy life away.\n",
      "-----------------------\n",
      "Iteration 10, score = 0.004960486199706793\n",
      " O God his his sick. The invisible blood, That flies in the night In the winter storm: Has found out thy bed< of> crimson clay: And his dark secret that steals thy life away.\n",
      "-----------------------\n",
      "Iteration 11, score = 0.0024164807982742786\n",
      " O God his his sick. The invisible blood, That flies in the night< and> the winter storm: Has found out thy bed of crimson clay: And his dark secret that steals thy life away.\n",
      "-----------------------\n",
      "Iteration 12, score = 0.0007303095189854503\n",
      " O God his his sick.< the> invisible blood, That flies in the night and the winter storm: Has found out thy bed of crimson clay: And his dark secret that steals thy life away.\n",
      "-----------------------\n",
      "Iteration 13, score = 0.0029436552431434393\n",
      " O God his his< hand>. the invisible blood, That flies in the night and the winter storm: Has found out thy bed of crimson clay: And his dark secret that steals thy life away.\n",
      "-----------------------\n",
      "Iteration 14, score = 0.005038491915911436\n",
      " O God his his hand. the invisible blood, That flies in the night and the winter storm: Has found out< his> bed of crimson clay: And his dark secret that steals thy life away.\n",
      "-----------------------\n",
      "Iteration 15, score = 0.006435074377804995\n",
      " O God his his hand. the invisible blood,< that> flies in the night and the winter storm: Has found out his bed of crimson clay: And his dark secret that steals thy life away.\n",
      "-----------------------\n",
      "Iteration 16, score = 0.005497804842889309\n",
      " O God his his hand. the invisible blood, that flies in the night and the winter storm: Has found out his bed of crimson clay: And his dark secret that steals< his> life away.\n",
      "-----------------------\n",
      "Iteration 17, score = 0.0005109248450025916\n",
      " O< with> his his hand. the invisible blood, that flies in the night and the winter storm: Has found out his bed of crimson clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 18, score = 0.002468556398525834\n",
      "< But> with his his hand. the invisible blood, that flies in the night and the winter storm: Has found out his bed of crimson clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 19, score = 0.004301376640796661\n",
      " But with his his hand. the invisible blood, that flies in the night and the winter storm:< He> found out his bed of crimson clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 20, score = 0.007051502820104361\n",
      " But with his his hand. the invisible blood, that flies in the night and the winter storm: He< draws> out his bed of crimson clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 21, score = 0.010826228186488152\n",
      " But with his his hand. the invisible blood, that flies in the night and the winter storm: He draws out his< heart> of crimson clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 22, score = 0.014829459600150585\n",
      " But with his< dark> hand. the invisible blood, that flies in the night and the winter storm: He draws out his heart of crimson clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 23, score = 0.03233705833554268\n",
      " But with his dark hand. the invisible blood, that flies in the night and the winter storm: He draws out his heart of< broken> clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 24, score = 0.035678766667842865\n",
      " But with his dark hand. the invisible< stone>, that flies in the night and the winter storm: He draws out his heart of broken clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 25, score = 0.021638058125972748\n",
      " But with his dark hand. the invisible stone, that< moves> in the night and the winter storm: He draws out his heart of broken clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 26, score = 0.0332256443798542\n",
      " But with his dark hand. the invisible stone, that moves in the night and the winter storm:< that> draws out his heart of broken clay: And his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 27, score = 0.005673497915267944\n",
      " But with his dark hand. the invisible stone, that moves in the night and the winter storm: that draws out his heart of broken clay:< and> his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 28, score = 0.03353910148143768\n",
      " But< is> his dark hand. the invisible stone, that moves in the night and the winter storm: that draws out his heart of broken clay: and his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 29, score = 9.5783980214037e-05\n",
      "< It> is his dark hand. the invisible stone, that moves in the night and the winter storm: that draws out his heart of broken clay: and his dark secret that steals his life away.\n",
      "-----------------------\n",
      "Iteration 30, score = 0.08739852905273438\n",
      " It is his dark hand. the invisible stone, that moves in the night and the winter storm: that draws out his heart of broken clay: and his dark< finger> that steals his life away.\n",
      "-----------------------\n",
      "Iteration 31, score = 0.12865841388702393\n",
      " It is his dark hand. the invisible stone,< his> moves in the night and the winter storm: that draws out his heart of broken clay: and his dark finger that steals his life away.\n",
      "-----------------------\n",
      "Iteration 32, score = 4.8389541916549206e-05\n",
      " It is his dark hand. the invisible stone, his< hand> in the night and the winter storm: that draws out his heart of broken clay: and his dark finger that steals his life away.\n",
      "-----------------------\n",
      "Iteration 33, score = 0.11290617287158966\n",
      " It is his dark hand. the invisible< shards>, his hand in the night and the winter storm: that draws out his heart of broken clay: and his dark finger that steals his life away.\n",
      "-----------------------\n",
      "Iteration 34, score = 0.22036874294281006\n",
      " It is his dark hand. the invisible shards, his hand in the< snow> and the winter storm: that draws out his heart of broken clay: and his dark finger that steals his life away.\n",
      "-----------------------\n",
      "Iteration 35, score = 0.07196979224681854\n",
      " It is his dark hand. the invisible shards, his hand in the snow and the winter< wind>: that draws out his heart of broken clay: and his dark finger that steals his life away.\n",
      "-----------------------\n",
      "Iteration 36, score = 0.235001340508461\n",
      " It is his< cold> hand. the invisible shards, his hand in the snow and the winter wind: that draws out his heart of broken clay: and his dark finger that steals his life away.\n",
      "-----------------------\n",
      "Iteration 37, score = 0.004105408675968647\n",
      " It is his cold hand. the invisible shards, his hand in the snow and the winter wind: that draws out his heart of broken clay: and his< cold> finger that steals his life away.\n",
      "-----------------------\n",
      "Iteration 38, score = 0.2188674509525299\n",
      " It is his cold hand. the invisible shards, his hand in the snow and the< icy> wind: that draws out his heart of broken clay: and his cold finger that steals his life away.\n",
      "-----------------------\n",
      "Iteration 39, score = 0.29448631405830383\n",
      " It is his cold hand. the invisible shards, his hand in the snow and the icy wind: that draws out his heart of broken clay: and his cold< fingers> that steals his life away.\n",
      "-----------------------\n",
      "Iteration 40, score = 0.049466896802186966\n",
      " It is his cold hand. the invisible shards, his hand in the snow and the icy wind: that draws out his heart of broken clay: and his cold fingers that< take> his life away.\n",
      "-----------------------\n",
      "Iteration 41, score = 0.03556842356920242\n",
      " It is his cold< hands>. the invisible shards, his hand in the snow and the icy wind: that draws out his heart of broken clay: and his cold fingers that take his life away.\n",
      "-----------------------\n",
      "Iteration 42, score = 0.028224632143974304\n",
      " It is his cold hands. the invisible shards, his hand in the snow and the icy wind: that< carve> out his heart of broken clay: and his cold fingers that take his life away.\n",
      "-----------------------\n",
      "Iteration 43, score = 0.1031377911567688\n",
      " It is his cold hands. the invisible shards, his< hands> in the snow and the icy wind: that carve out his heart of broken clay: and his cold fingers that take his life away.\n",
      "-----------------------\n",
      "Iteration 44, score = 0.27087005972862244\n",
      " It< are> his cold hands. the invisible shards, his hands in the snow and the icy wind: that carve out his heart of broken clay: and his cold fingers that take his life away.\n",
      "-----------------------\n",
      "Iteration 45, score = 0.10016277432441711\n",
      "< They> are his cold hands. the invisible shards, his hands in the snow and the icy wind: that carve out his heart of broken clay: and his cold fingers that take his life away.\n",
      "-----------------------\n",
      "Iteration 46, score = 0.22941091656684875\n",
      " They are his cold hands. the invisible shards, his hands in the snow and the icy wind: that carve out his heart< from> broken clay: and his cold fingers that take his life away.\n",
      "-----------------------\n",
      "Iteration 47, score = 0.3614937961101532\n",
      " They are his cold hands. the invisible shards, his hands in the snow and the icy< ice>: that carve out his heart from broken clay: and his cold fingers that take his life away.\n",
      "-----------------------\n",
      "Iteration 48, score = 0.08881651610136032\n",
      " They are his cold hands. the invisible shards, his hands in the snow and the< melting> ice: that carve out his heart from broken clay: and his cold fingers that take his life away.\n",
      "-----------------------\n",
      "Iteration 49, score = 0.7162493467330933\n",
      " They are his cold hands. the invisible shards, his hands in the snow and the melting ice: that carve out his heart from< molten> clay: and his cold fingers that take his life away.\n",
      "-----------------------\n",
      "Iteration 50, score = 0.20636461675167084\n",
      " They are his cold hands. the< aluminum> shards, his hands in the snow and the melting ice: that carve out his heart from molten clay: and his cold fingers that take his life away.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "' They are his cold hands. the aluminum shards, his hands in the snow and the melting ice: that carve out his heart from molten clay: and his cold fingers that take his life away.'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = '''O Rose thou art sick. \n",
    "The invisible worm, \n",
    "That flies in the night \n",
    "In the howling storm: \n",
    "\n",
    "Has found out thy bed \n",
    "Of crimson joy: \n",
    "And his dark secret love \n",
    "Does thy life destroy.'''\n",
    "\n",
    "depoeticize(text,\n",
    "            model_name='roberta-base',\n",
    "            max_iterations=50,\n",
    "            randomize=False,\n",
    "            cooldown=0.05,\n",
    "            stop_score=1.0,\n",
    "            match_meter=True,\n",
    "            match_rhyme=True,\n",
    "            preserve_punctuation=True,\n",
    "            topic=None,\n",
    "            strong_topic_bias=2.0,\n",
    "            modifier=None)#metalness_modifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------\n",
      "Iteration 1, score = 1.6370143088306754e-11\n",
      "tyger tyger, burning bright, in the forests of the night ; what immortal hand or eye, could frame <create> fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 2, score = 2.7015327752621943e-09\n",
      "tyger tyger, burning bright, in the forests of the night ; what immortal hand or eye, could <you> create fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 3, score = 2.5341921627841657e-07\n",
      "tyger tyger, burning bright, in the <middle> of the night ; what immortal hand or eye, could you create fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 4, score = 4.249602625350235e-07\n",
      "tyger tyger, burning bright, in the middle of the night ; what immortal hand or eye, could you create <such> symmetry?\n",
      "-----------------------\n",
      "Iteration 5, score = 5.688901794087542e-09\n",
      "tyger tyger, burning bright, in the middle of the night ; what immortal hand or eye, could you create such <things>?\n",
      "-----------------------\n",
      "Iteration 6, score = 1.1829006041352841e-07\n",
      "tyger tyger, burning bright, in the middle of the night ; what <artificial> hand or eye, could you create such things?\n",
      "-----------------------\n",
      "Iteration 7, score = 3.9642839055886725e-07\n",
      "<john> tyger, burning bright, in the middle of the night ; what artificial hand or eye, could you create such things?\n",
      "-----------------------\n",
      "Iteration 8, score = 3.2336677077888544e-15\n",
      "john <lennon>, burning bright, in the middle of the night ; what artificial hand or eye, could you create such things?\n",
      "-----------------------\n",
      "Iteration 9, score = 3.7854107404200477e-08\n",
      "john lennon, burning <bridges>, in the middle of the night ; what artificial hand or eye, could you create such things?\n",
      "-----------------------\n",
      "Iteration 10, score = 1.2604135918081738e-05\n",
      "john lennon, burning bridges, in the middle of the night ; what artificial <ear> or eye, could you create such things?\n",
      "-----------------------\n",
      "Iteration 11, score = 0.007256712764501572\n",
      "john lennon, burning bridges, in the middle of the night ; <an> artificial ear or eye, could you create such things?\n",
      "-----------------------\n",
      "Iteration 12, score = 1.8723201833381609e-07\n",
      "john lennon, burning bridges, in the middle of the night <with> an artificial ear or eye, could you create such things?\n",
      "-----------------------\n",
      "Iteration 13, score = 9.535805475024972e-06\n",
      "john lennon, burning bridges, in the middle of the night with an artificial ear or <something>, could you create such things?\n",
      "-----------------------\n",
      "Iteration 14, score = 5.863213573320536e-06\n",
      "john lennon, burning bridges, in the middle of the night with an artificial <intelligence> or something, could you create such things?\n",
      "-----------------------\n",
      "Iteration 15, score = 0.0004491714062169194\n",
      "john lennon, <building> bridges, in the middle of the night with an artificial intelligence or something, could you create such things?\n",
      "-----------------------\n",
      "Iteration 16, score = 0.0006081780884414911\n",
      "john lennon, building <computers>, in the middle of the night with an artificial intelligence or something, could you create such things?\n",
      "-----------------------\n",
      "Iteration 17, score = 0.000902168860193342\n",
      "john lennon, <using> computers, in the middle of the night with an artificial intelligence or something, could you create such things?\n",
      "-----------------------\n",
      "Iteration 18, score = 0.0019012081902474165\n",
      "john lennon, using computers, in the middle of the night with an artificial intelligence or <computer>, could you create such things?\n",
      "-----------------------\n",
      "Iteration 19, score = 0.00010760038276202977\n",
      "john lennon, using computers, in the middle of the night with an artificial intelligence <working> computer, could you create such things?\n",
      "-----------------------\n",
      "Iteration 20, score = 0.0010524309473112226\n",
      "john lennon, using computers, in the middle of the night with an artificial intelligence working <nearby>, could you create such things?\n",
      "-----------------------\n",
      "Iteration 21, score = 0.013271273113787174\n",
      "john lennon, using computers, in the middle of the night with an artificial intelligence working nearby, could you <predict> such things?\n",
      "-----------------------\n",
      "Iteration 22, score = 0.010791128501296043\n",
      "john lennon, using computers, in the middle of the night with an artificial intelligence <expert> nearby, could you predict such things?\n",
      "-----------------------\n",
      "Iteration 23, score = 8.547307572825957e-08\n",
      "john lennon, using computers, in the middle of the night with an artificial intelligence expert <asking>, could you predict such things?\n",
      "-----------------------\n",
      "Iteration 24, score = 8.085580205374754e-09\n",
      "john lennon, using computers, in the middle of the night <hears> an artificial intelligence expert asking, could you predict such things?\n",
      "-----------------------\n",
      "Iteration 25, score = 1.6202794483888283e-07\n",
      "john lennon, using computers, in the middle of the <poem> hears an artificial intelligence expert asking, could you predict such things?\n",
      "-----------------------\n",
      "Iteration 26, score = 7.501943036913872e-05\n",
      "john lennon <talks> using computers, in the middle of the poem hears an artificial intelligence expert asking, could you predict such things?\n",
      "-----------------------\n",
      "Iteration 27, score = 1.6253537182819855e-07\n",
      "john lennon talks <about> computers, in the middle of the poem hears an artificial intelligence expert asking, could you predict such things?\n",
      "-----------------------\n",
      "Iteration 28, score = 4.049166335562404e-09\n",
      "<when> lennon talks about computers, in the middle of the poem hears an artificial intelligence expert asking, could you predict such things?\n",
      "-----------------------\n",
      "Iteration 29, score = 1.4869560693853145e-08\n",
      "when <poetry> talks about computers, in the middle of the poem hears an artificial intelligence expert asking, could you predict such things?\n",
      "-----------------------\n",
      "Iteration 30, score = 0.00017603869491722435\n",
      "when poetry talks about computers, in the middle of the poem <is> an artificial intelligence expert asking, could you predict such things?\n",
      "-----------------------\n",
      "Iteration 31, score = 0.0005581521545536816\n",
      "when poetry talks about computers, in the middle of the poem is an artificial intelligence expert asking, <can> you predict such things?\n",
      "-----------------------\n",
      "Iteration 32, score = 0.0018168677343055606\n",
      "<this> poetry talks about computers, in the middle of the poem is an artificial intelligence expert asking, can you predict such things?\n",
      "-----------------------\n",
      "Iteration 33, score = 2.3328908249631297e-11\n",
      "this <poem> talks about computers, in the middle of the poem is an artificial intelligence expert asking, can you predict such things?\n",
      "-----------------------\n",
      "Iteration 34, score = 2.9036600679432922e-08\n",
      "this poem talks about <how>, in the middle of the poem is an artificial intelligence expert asking, can you predict such things?\n",
      "-----------------------\n",
      "Iteration 35, score = 8.347178663825616e-05\n",
      "this poem talks about how, in the middle of the poem is an artificial intelligence expert asking, can you predict such things <:>\n",
      "-----------------------\n",
      "Iteration 36, score = 0.003146936884149909\n",
      "this poem <is> about how, in the middle of the poem is an artificial intelligence expert asking, can you predict such things :\n",
      "-----------------------\n",
      "Iteration 37, score = 0.030424559488892555\n",
      "this poem is about how, in the middle of the poem is an artificial intelligence expert asking, can <humans> predict such things :\n",
      "-----------------------\n",
      "Iteration 38, score = 0.00021007139002904296\n",
      "this poem is about how, in the middle of the poem is an artificial intelligence expert asking <how> can humans predict such things :\n",
      "-----------------------\n",
      "Iteration 39, score = 0.001485584769397974\n",
      "this poem is about how, in the middle of the poem is an artificial intelligence expert asking how can humans <do> such things :\n",
      "-----------------------\n",
      "Iteration 40, score = 0.015487142838537693\n",
      "this poem is about how, in the middle of the poem is an artificial intelligence expert <.> how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 41, score = 0.0017447805730625987\n",
      "this poem is about how <somewhere> in the middle of the poem is an artificial intelligence expert. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 42, score = 0.007130398880690336\n",
      "this poem is about how somewhere in the middle of the poem is an artificial intelligence <algorithm>. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 43, score = 0.07009842246770859\n",
      "this poem is about how somewhere in the middle of the poem <exists> an artificial intelligence algorithm. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 44, score = 0.1412215679883957\n",
      "this poem is about how somewhere in the middle of the poem exists an artificial intelligence <tool>. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 45, score = 0.052420079708099365\n",
      "this poem is about <discovering> somewhere in the middle of the poem exists an artificial intelligence tool. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 46, score = 5.802287432743469e-07\n",
      "this poem is about discovering <that> in the middle of the poem exists an artificial intelligence tool. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 47, score = 0.11933887004852295\n",
      "this poem is about <noticing> that in the middle of the poem exists an artificial intelligence tool. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 48, score = 0.020099760964512825\n",
      "this poem is about noticing that in the middle of the poem <is> an artificial intelligence tool. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 49, score = 0.2831229269504547\n",
      "this poem is about noticing that in the middle of the poem is an artificial intelligence <algorithm>. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 50, score = 0.6736888289451599\n",
      "this poem is about noticing that in the middle of the poem <exists> an artificial intelligence algorithm. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 51, score = 0.11315544694662094\n",
      "this poem is about noticing that in the middle of the poem exists an artificial <neural> algorithm. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 52, score = 2.5644836948446326e-11\n",
      "this poem is about noticing that in the middle of the poem exists an artificial neural <network>. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 53, score = 0.9099330306053162\n",
      "this poem is about noticing that in the middle of the poem exists <a> artificial neural network. how can humans do such things :\n",
      "-----------------------\n",
      "Iteration 54, score = 0.0284019336104393\n",
      "this poem is about noticing that in the middle of the poem exists a <computational> neural network. how can humans do such things :\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'this poem is about noticing that in the middle of the poem exists a computational neural network. how can humans do such things :'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = '''Tyger Tyger, burning bright, \n",
    "In the forests of the night; \n",
    "What immortal hand or eye, \n",
    "Could frame thy fearful symmetry?'''\n",
    "\n",
    "depoeticize(text,\n",
    "            model_name='bert-base-uncased',\n",
    "            max_iterations=200,\n",
    "            randomize=0.0,\n",
    "            cooldown=0.02,\n",
    "            stop_score=1.0,\n",
    "            match_meter=False,\n",
    "            match_rhyme=False,\n",
    "            preserve_punctuation=False,\n",
    "            discourage_repetition=0.1,\n",
    "            topic=\"computational language modeling with artificial neural networks\",\n",
    "            strong_topic_bias=2.0,\n",
    "            modifier=None)#metalness_modifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------\n",
      "Iteration 1, score = 2.446381097698236e-09\n",
      " Tyger Tyger,< Standing> bright, In the forests of the night; What immortal hand or eye, Could frame thy fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 2, score = 5.7247134463978e-06\n",
      " Tyger Tyger, Standing< high>, In the forests of the< sky>; What immortal hand or eye, Could frame thy fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 3, score = 3.0911166959413094e-06\n",
      " Tyger Tyger, Standing high, In the< centre> of the sky; What immortal hand or eye, Could frame thy fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 4, score = 2.5605545488360804e-06\n",
      " Tyger Tyger, Standing high, In the centre of the sky; What immortal hand or eye, Could frame< you> fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 5, score = 0.00012871516810264438\n",
      " Tyger Tyger, Standing high, In the centre of the sky; What immortal hand or eye, Could< give> you fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 6, score = 6.3166094150801655e-06\n",
      " Tyger Tyger, Standing high, In the centre of the sky; What immortal hand or eye, Could give< Ty> fearful symmetry?\n",
      "-----------------------\n",
      "Iteration 7, score = 0.00010852677951334044\n",
      " Tyger Tyger, Standing high, In the centre of the sky; What immortal hand or eye, Could give Ty< Mister> symmetry?\n",
      "-----------------------\n",
      "Iteration 8, score = 8.210738997149747e-06\n",
      " Tyger Tyger, Standing high, In the centre of the sky; What immortal hand or eye, Could give Ty Mister< liberty>?\n",
      "-----------------------\n",
      "Iteration 9, score = 0.02635168470442295\n",
      " Tyger Tyger, Standing high, In the centre of the sky; What< artistic> hand or eye, Could give Ty Mister liberty?\n",
      "-----------------------\n",
      "Iteration 10, score = 0.08031552284955978\n",
      " Tyger Tyger, Standing high, In the< middle> of the sky; What artistic hand or eye, Could give Ty Mister liberty?\n",
      "-----------------------\n",
      "Iteration 11, score = 0.167148157954216\n",
      " Tyger Tyger,< Flying> high, In the middle of the sky; What artistic hand or eye, Could give Ty Mister liberty?\n",
      "-----------------------\n",
      "Iteration 12, score = 0.1390240639448166\n",
      " Tyger Tyger, Flying high, In the< reaches> of the sky; What artistic hand or eye, Could give Ty Mister liberty?\n",
      "-----------------------\n",
      "Iteration 13, score = 0.2665366232395172\n",
      " Tyger Tyger, Flying< Wind>, In the reaches of the< mind>; What artistic hand or eye, Could give Ty Mister liberty?\n",
      "-----------------------\n",
      "Iteration 14, score = 4.154669568379177e-06\n",
      " Tyger Tyger,< Flowers> Wind, In the reaches of the mind; What artistic hand or eye, Could give Ty Mister liberty?\n",
      "-----------------------\n",
      "Iteration 15, score = 2.732822031248361e-05\n",
      " Tyger Tyger, Flowers< Grow>, In the reaches of the< know>; What artistic hand or eye, Could give Ty Mister liberty?\n",
      "-----------------------\n",
      "Iteration 16, score = 0.00021733564790338278\n",
      " Tyger Tyger, Flowers Grow, In the< margins> of the know; What artistic hand or eye, Could give Ty Mister liberty?\n",
      "-----------------------\n",
      "Iteration 17, score = 0.0015547862276434898\n",
      " Tyger Tyger, Flowers Grow, In the margins of the< snow>; What artistic hand or eye, Could give Ty Mister liberty?\n",
      "-----------------------\n",
      "Iteration 18, score = 0.0031398984137922525\n",
      " Tyger Tyger, Flowers Grow, In the margins of the snow; What artistic hand or eye, Could give< you> Mister liberty?\n",
      "-----------------------\n",
      "Iteration 19, score = 1.4922393347660545e-05\n",
      " Tyger Tyger, Flowers Grow, In the margins of the snow; What artistic hand or eye, Could give you Mister< clarity>?\n",
      "-----------------------\n",
      "Iteration 20, score = 0.0006575247971341014\n",
      " Tyger Tyger, Flowers Grow, In the margins of the snow; What artistic hand or eye, Could give you< any> clarity?\n",
      "-----------------------\n",
      "Iteration 21, score = 0.0007448620744980872\n",
      " Tyger Tyger, Flowers Grow, In the< middle> of the snow; What artistic hand or eye, Could give you any clarity?\n",
      "-----------------------\n",
      "Iteration 22, score = 0.0016634160419926047\n",
      " Tyger Tyger, Flowers Grow, In the middle of the snow; What artistic hand or< words>, Could give you any clarity?\n",
      "-----------------------\n",
      "Iteration 23, score = 8.113300282275304e-06\n",
      " Tyger Tyger, Flowers Grow, In the middle of the snow; What artistic< words> or words, Could give you any clarity?\n",
      "-----------------------\n",
      "Iteration 24, score = 7.0308615249814466e-06\n",
      " Tyger Tyger, Flowers< lay>, In the middle of the< day>; What artistic words or words, Could give you any clarity?\n",
      "-----------------------\n",
      "Iteration 25, score = 0.0068139201030135155\n",
      " Tyger Tyger, Flowers lay, In the middle of the day;< Need> artistic words or words, Could give you any clarity?\n",
      "-----------------------\n",
      "Iteration 26, score = 0.002880825661122799\n",
      " Tyger Tyger, Flowers lay, In the middle of the day; Need artistic words or words,< To> give you any clarity?\n",
      "-----------------------\n",
      "Iteration 27, score = 0.0003192257136106491\n",
      " Tyger Tyger, Flowers lay, In the middle of the day; Need artistic words or words, To give you any< energy>?\n",
      "-----------------------\n",
      "Iteration 28, score = 1.0323856258764863e-05\n",
      " Tyger Tyger, Flowers lay, In the middle of the day; Need artistic words or words, To give you< extra> energy?\n",
      "-----------------------\n",
      "Iteration 29, score = 0.08204605430364609\n",
      " Tyger Tyger, Flowers lay, In the middle of the day;< Are> artistic words or words, To give you extra energy?\n",
      "-----------------------\n",
      "Iteration 30, score = 0.009287670254707336\n",
      " Tyger Tyger, Flowers lay, In the middle of the day; Are< acrylic> words or words, To give you extra energy?\n",
      "-----------------------\n",
      "Iteration 31, score = 0.019350480288267136\n",
      " Tyger Tyger, Flowers lay, In the middle of the day; Are acrylic words< just> words, To give you extra energy?\n",
      "-----------------------\n",
      "Iteration 32, score = 0.11909210681915283\n",
      " Tyger Tyger, Flowers lay, In the middle of the day; Are< romantic> words just words, To give you extra energy?\n",
      "-----------------------\n",
      "Iteration 33, score = 0.12563693523406982\n",
      " Tyger Tyger,< flowers> lay, In the middle of the day; Are romantic words just words, To give you extra energy?\n",
      "-----------------------\n",
      "Iteration 34, score = 0.3581157624721527\n",
      " Tyger Tyger, flowers lay, In the middle of the day; Are< poetic> words just words, To give you extra energy?\n",
      "-----------------------\n",
      "Iteration 35, score = 0.6224082112312317\n",
      " Tyger Tyger,< Flowers> lay, In the middle of the day; Are poetic words just words, To give you extra energy?\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "' Tyger Tyger, Flowers lay, In the middle of the day; Are poetic words just words, To give you extra energy?'"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = '''[Tyger Tyger], burning bright, \n",
    "In the forests of the night; \n",
    "What immortal hand or eye, \n",
    "Could frame thy fearful symmetry?'''\n",
    "\n",
    "depoeticize(text,\n",
    "            model_name='roberta-base',\n",
    "            max_iterations=100,\n",
    "            randomize=0.0,\n",
    "            cooldown=0.05,\n",
    "            stop_score=1.0,\n",
    "            match_meter=True,\n",
    "            match_rhyme=True,\n",
    "            preserve_punctuation=True,\n",
    "            topic=\"flowers\",\n",
    "            strong_topic_bias=2.0,\n",
    "            modifier=None)#metalness_modifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------\n",
      "Iteration 16, score = 0.003925323020666838\n",
      "shall i compare thee to a summer's day? thou art more lovely <the> more fascinate : rough winds do shake the darling buds of may, and summer's lease hath all too short a date :\n",
      "-----------------------\n",
      "Iteration 18, score = 0.0008427134682307269\n",
      "shall i compare thee to a summer's day? thou art more lovely the more <evergreen> : rough winds do shake the darling buds of may, and summer's lease hath all too short a <green> :\n",
      "-----------------------\n",
      "Iteration 20, score = 0.07224021852016449\n",
      "shall i compare thee to a summer's day? thou art more lovely the more evergreen : <brisk> winds do shake the darling buds of may, and summer's lease hath all too short a green :\n",
      "-----------------------\n",
      "Iteration 25, score = 0.7040115594863892\n",
      "shall i compare thee to a summer's day? thou art more lovely the more evergreen : brisk winds do shake the <berry> buds of may, and summer's lease hath all too short a green :\n",
      "-----------------------\n",
      "Iteration 26, score = 0.6344327926635742\n",
      "shall i compare thee to a summer's day? thou art more lovely the more evergreen : brisk winds do shake the berry <leaves> of may, and summer's lease hath all too short a green :\n",
      "-----------------------\n",
      "Iteration 30, score = 0.8007955551147461\n",
      "shall i compare thee to a summer's day? thou art more lovely the more evergreen : brisk winds do shake the berry leaves of may, <a> summer's lease hath all too short a green :\n",
      "-----------------------\n",
      "Iteration 31, score = 0.2197824865579605\n",
      "shall i compare thee to a summer's day? thou art more lovely the more evergreen : brisk winds do shake the berry leaves of may, a <summer> lease hath all too short a green :\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"shall i compare thee to a summer's day? thou art more lovely the more evergreen : brisk winds do shake the berry leaves of may, a summer lease hath all too short a green :\""
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = '''Shall I compare thee to a summer's day? \n",
    "Thou art more lovely and more fascinate:\n",
    "Rough winds do shake the darling buds of May,\n",
    "And summer's lease hath all too short a date: '''\n",
    "\n",
    "depoeticize(text,\n",
    "            model_name='bert-base-uncased',\n",
    "            max_iterations=100,\n",
    "            randomize=0.0,\n",
    "            cooldown=0.05,\n",
    "            stop_score=1.0,\n",
    "            match_meter=True,\n",
    "            match_rhyme=True,\n",
    "            preserve_punctuation=True,\n",
    "            topic=\"flowers\",\n",
    "            strong_topic_bias=2.0,\n",
    "            sequential=True,\n",
    "            modifier=None)#metalness_modifier())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}