From 1957616b27f94388b1a375522470327350dd149a Mon Sep 17 00:00:00 2001 From: Yanjun Qi Date: Mon, 11 Sep 2023 11:01:08 -0400 Subject: [PATCH 1/2] Fixing the pytest attack command error.. The errors were caused by the previous pull request --- docs/2notebook/0_End_to_End.ipynb | 2914 +++++------ .../1_Introduction_and_Transformations.ipynb | 2 +- docs/2notebook/2_Constraints.ipynb | 17 +- docs/2notebook/3_Augmentations.ipynb | 164 +- .../4_Custom_Datasets_Word_Embedding.ipynb | 25 +- docs/2notebook/Example_6_Chinese_Attack.ipynb | 4336 ++++++++--------- tests/test_attacked_text.py | 2 +- tests/test_command_line/test_loggers.py | 101 + tests/test_word_embedding.py | 4 +- textattack/attack.py | 4 +- textattack/attack_args.py | 4 +- textattack/constraints/grammaticality/cola.py | 2 +- .../google_language_model/alzantot_goog_lm.py | 2 +- .../grammaticality/part_of_speech.py | 2 +- .../sentence_encoders/thought_vector.py | 2 +- textattack/goal_functions/goal_function.py | 5 +- .../goal_functions/text/minimize_bleu.py | 2 +- .../text/non_overlapping_output.py | 4 +- .../metrics/attack_metrics/words_perturbed.py | 2 +- 19 files changed, 3884 insertions(+), 3710 deletions(-) create mode 100644 tests/test_command_line/test_loggers.py diff --git a/docs/2notebook/0_End_to_End.ipynb b/docs/2notebook/0_End_to_End.ipynb index 0231b44c7..d4388e156 100644 --- a/docs/2notebook/0_End_to_End.ipynb +++ b/docs/2notebook/0_End_to_End.ipynb @@ -1,1440 +1,1500 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.11" - }, - "colab": { - "name": "0_End_to_End.ipynb", - "provenance": [], - "collapsed_sections": [] - }, - "accelerator": "GPU" + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "-e84g1YoseoE" + }, + "source": [ + "# TextAttack End-to-End\n", + "\n", + "This tutorial provides a broad end-to-end overview of training, evaluating, and attacking a model using TextAttack." + ] }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "-e84g1YoseoE" - }, - "source": [ - "# TextAttack End-to-End\n", - "\n", - "This tutorial provides a broad end-to-end overview of training, evaluating, and attacking a model using TextAttack." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "pGv59SZzseoG" - }, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/QData/TextAttack/blob/master/docs/2notebook/0_End_to_End.ipynb)\n", - "\n", - "[![View Source on GitHub](https://img.shields.io/badge/github-view%20source-black.svg)](https://github.com/QData/TextAttack/blob/master/docs/2notebook/0_End_to_End.ipynb)" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "AQTkpf9RslEA", - "outputId": "20172655-e6ce-46c0-ba57-cb3a7f955ee3" - }, - "source": [ - "!pip3 install textattack[tensorflow]" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Collecting textattack[tensorflow]\n", - " Downloading textattack-0.3.3-py3-none-any.whl (361 kB)\n", - "\u001b[K |████████████████████████████████| 361 kB 4.1 MB/s \n", - "\u001b[?25hCollecting language-tool-python\n", - " Downloading language_tool_python-2.6.1-py3-none-any.whl (30 kB)\n", - "Collecting lemminflect\n", - " Downloading lemminflect-0.2.2-py3-none-any.whl (769 kB)\n", - "\u001b[K |████████████████████████████████| 769 kB 36.0 MB/s \n", - "\u001b[?25hCollecting word2number\n", - " Downloading word2number-1.1.zip (9.7 kB)\n", - "Collecting transformers>=3.3.0\n", - " Downloading transformers-4.11.3-py3-none-any.whl (2.9 MB)\n", - "\u001b[K |████████████████████████████████| 2.9 MB 37.9 MB/s \n", - "\u001b[?25hCollecting flair\n", - " Downloading flair-0.9-py3-none-any.whl (319 kB)\n", - "\u001b[K |████████████████████████████████| 319 kB 50.4 MB/s \n", - "\u001b[?25hRequirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.4.1)\n", - "Collecting terminaltables\n", - " Downloading terminaltables-3.1.0.tar.gz (12 kB)\n", - "Requirement already satisfied: more-itertools in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (8.10.0)\n", - "Requirement already satisfied: editdistance in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (0.5.3)\n", - "Collecting bert-score>=0.3.5\n", - " Downloading bert_score-0.3.10-py3-none-any.whl (59 kB)\n", - "\u001b[K |████████████████████████████████| 59 kB 7.1 MB/s \n", - "\u001b[?25hRequirement already satisfied: torch!=1.8,>=1.7.0 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.9.0+cu111)\n", - "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (3.2.5)\n", - "Collecting num2words\n", - " Downloading num2words-0.5.10-py3-none-any.whl (101 kB)\n", - "\u001b[K |████████████████████████████████| 101 kB 11.5 MB/s \n", - "\u001b[?25hCollecting datasets\n", - " Downloading datasets-1.13.0-py3-none-any.whl (285 kB)\n", - "\u001b[K |████████████████████████████████| 285 kB 53.3 MB/s \n", - "\u001b[?25hCollecting lru-dict\n", - " Downloading lru-dict-1.1.7.tar.gz (10 kB)\n", - "Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.1.5)\n", - "Collecting tqdm<4.50.0,>=4.27\n", - " Downloading tqdm-4.49.0-py2.py3-none-any.whl (69 kB)\n", - "\u001b[K |████████████████████████████████| 69 kB 7.5 MB/s \n", - "\u001b[?25hRequirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.7.1)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (3.3.0)\n", - "Requirement already satisfied: numpy>=1.19.2 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.19.5)\n", - "Requirement already satisfied: tensorflow>=2 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (2.6.0)\n", - "Requirement already satisfied: tensorflow-hub in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (0.12.0)\n", - "Collecting tensorboardX\n", - " Downloading tensorboardX-2.4-py2.py3-none-any.whl (124 kB)\n", - "\u001b[K |████████████████████████████████| 124 kB 45.0 MB/s \n", - "\u001b[?25hCollecting tensorflow-text>=2\n", - " Downloading tensorflow_text-2.6.0-cp37-cp37m-manylinux1_x86_64.whl (4.4 MB)\n", - "\u001b[K |████████████████████████████████| 4.4 MB 38.7 MB/s \n", - "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from bert-score>=0.3.5->textattack[tensorflow]) (3.2.2)\n", - "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.7/dist-packages (from bert-score>=0.3.5->textattack[tensorflow]) (21.0)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from bert-score>=0.3.5->textattack[tensorflow]) (2.23.0)\n", - "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.9->bert-score>=0.3.5->textattack[tensorflow]) (2.4.7)\n", - "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.0.1->textattack[tensorflow]) (2.8.2)\n", - "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.0.1->textattack[tensorflow]) (2018.9)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas>=1.0.1->textattack[tensorflow]) (1.15.0)\n", - "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.12)\n", - "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (0.12.0)\n", - "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (0.2.0)\n", - "Requirement already satisfied: h5py~=3.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (3.1.0)\n", - "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (3.7.4.3)\n", - "Requirement already satisfied: tensorflow-estimator~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (2.6.0)\n", - "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.1.0)\n", - "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.12.1)\n", - "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.1.2)\n", - "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.6.3)\n", - "Requirement already satisfied: gast==0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (0.4.0)\n", - "Requirement already satisfied: tensorboard~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (2.6.0)\n", - "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (3.17.3)\n", - "Requirement already satisfied: clang~=5.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (5.0)\n", - "Requirement already satisfied: keras~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (2.6.0)\n", - "Requirement already satisfied: grpcio<2.0,>=1.37.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.41.0)\n", - "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (0.37.0)\n", - "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (3.3.0)\n", - "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py~=3.1.0->tensorflow>=2->textattack[tensorflow]) (1.5.2)\n", - "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (1.35.0)\n", - "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (3.3.4)\n", - "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (0.4.6)\n", - "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (57.4.0)\n", - "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (0.6.1)\n", - "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (1.8.0)\n", - "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (1.0.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (4.7.2)\n", - "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (4.2.4)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (0.2.8)\n", - "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (1.3.0)\n", - "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (4.8.1)\n", - "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (0.4.8)\n", - "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (1.24.3)\n", - "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (2.10)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (2021.5.30)\n", - "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (3.0.4)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (3.1.1)\n", - "Collecting sacremoses\n", - " Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n", - "\u001b[K |████████████████████████████████| 895 kB 38.8 MB/s \n", - "\u001b[?25hCollecting pyyaml>=5.1\n", - " Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n", - "\u001b[K |████████████████████████████████| 636 kB 30.6 MB/s \n", - "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers>=3.3.0->textattack[tensorflow]) (2019.12.20)\n", - "Collecting huggingface-hub>=0.0.17\n", - " Downloading huggingface_hub-0.0.19-py3-none-any.whl (56 kB)\n", - "\u001b[K |████████████████████████████████| 56 kB 5.2 MB/s \n", - "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n", - " Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n", - "\u001b[K |████████████████████████████████| 3.3 MB 39.6 MB/s \n", - "\u001b[?25hCollecting xxhash\n", - " Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n", - "\u001b[K |████████████████████████████████| 243 kB 54.2 MB/s \n", - "\u001b[?25hCollecting aiohttp\n", - " Downloading aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)\n", - "\u001b[K |████████████████████████████████| 1.3 MB 49.0 MB/s \n", - "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets->textattack[tensorflow]) (0.70.12.2)\n", - "Collecting fsspec[http]>=2021.05.0\n", - " Downloading fsspec-2021.10.0-py3-none-any.whl (125 kB)\n", - "\u001b[K |████████████████████████████████| 125 kB 44.3 MB/s \n", - "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets->textattack[tensorflow]) (0.3.4)\n", - "Collecting datasets\n", - " Downloading datasets-1.12.1-py3-none-any.whl (270 kB)\n", - "\u001b[K |████████████████████████████████| 270 kB 52.8 MB/s \n", - "\u001b[?25h Downloading datasets-1.12.0-py3-none-any.whl (269 kB)\n", - "\u001b[K |████████████████████████████████| 269 kB 51.8 MB/s \n", - "\u001b[?25h Downloading datasets-1.11.0-py3-none-any.whl (264 kB)\n", - "\u001b[K |████████████████████████████████| 264 kB 50.7 MB/s \n", - "\u001b[?25hRequirement already satisfied: pyarrow!=4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets->textattack[tensorflow]) (3.0.0)\n", - "Collecting sqlitedict>=1.6.0\n", - " Downloading sqlitedict-1.7.0.tar.gz (28 kB)\n", - "Collecting more-itertools\n", - " Downloading more_itertools-8.8.0-py3-none-any.whl (48 kB)\n", - "\u001b[K |████████████████████████████████| 48 kB 5.4 MB/s \n", - "\u001b[?25hCollecting conllu>=4.0\n", - " Downloading conllu-4.4.1-py2.py3-none-any.whl (15 kB)\n", - "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (0.22.2.post1)\n", - "Collecting segtok>=1.5.7\n", - " Downloading segtok-1.5.10.tar.gz (25 kB)\n", - "Collecting langdetect\n", - " Downloading langdetect-1.0.9.tar.gz (981 kB)\n", - "\u001b[K |████████████████████████████████| 981 kB 30.2 MB/s \n", - "\u001b[?25hRequirement already satisfied: gensim<=3.8.3,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (3.6.0)\n", - "Collecting sentencepiece==0.1.95\n", - " Downloading sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2 MB)\n", - "\u001b[K |████████████████████████████████| 1.2 MB 39.0 MB/s \n", - "\u001b[?25hCollecting gdown==3.12.2\n", - " Downloading gdown-3.12.2.tar.gz (8.2 kB)\n", - " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", - " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", - " Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n", - "Collecting mpld3==0.3\n", - " Downloading mpld3-0.3.tar.gz (788 kB)\n", - "\u001b[K |████████████████████████████████| 788 kB 33.7 MB/s \n", - "\u001b[?25hCollecting bpemb>=0.3.2\n", - " Downloading bpemb-0.3.3-py3-none-any.whl (19 kB)\n", - "Collecting janome\n", - " Downloading Janome-0.4.1-py2.py3-none-any.whl (19.7 MB)\n", - "\u001b[K |████████████████████████████████| 19.7 MB 50 kB/s \n", - "\u001b[?25hRequirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (0.8.9)\n", - "Requirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (0.1.2)\n", - "Collecting ftfy\n", - " Downloading ftfy-6.0.3.tar.gz (64 kB)\n", - "\u001b[K |████████████████████████████████| 64 kB 2.9 MB/s \n", - "\u001b[?25hCollecting deprecated>=1.2.4\n", - " Downloading Deprecated-1.2.13-py2.py3-none-any.whl (9.6 kB)\n", - "Collecting konoha<5.0.0,>=4.0.0\n", - " Downloading konoha-4.6.5-py3-none-any.whl (20 kB)\n", - "Collecting wikipedia-api\n", - " Downloading Wikipedia-API-0.5.4.tar.gz (18 kB)\n", - "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (4.2.6)\n", - "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim<=3.8.3,>=3.4.0->flair->textattack[tensorflow]) (5.2.1)\n", - "Requirement already satisfied: pymongo in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair->textattack[tensorflow]) (3.12.0)\n", - "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair->textattack[tensorflow]) (0.16.0)\n", - "Requirement already satisfied: networkx in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair->textattack[tensorflow]) (2.6.3)\n", - "Collecting importlib-metadata\n", - " Downloading importlib_metadata-3.10.1-py3-none-any.whl (14 kB)\n", - "Collecting requests\n", - " Downloading requests-2.26.0-py2.py3-none-any.whl (62 kB)\n", - "\u001b[K |████████████████████████████████| 62 kB 851 kB/s \n", - "\u001b[?25hCollecting overrides<4.0.0,>=3.0.0\n", - " Downloading overrides-3.1.0.tar.gz (11 kB)\n", - "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (3.6.0)\n", - "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->bert-score>=0.3.5->textattack[tensorflow]) (0.10.0)\n", - "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->bert-score>=0.3.5->textattack[tensorflow]) (1.3.2)\n", - "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (2.0.6)\n", - "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->flair->textattack[tensorflow]) (1.0.1)\n", - "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from ftfy->flair->textattack[tensorflow]) (0.2.5)\n", - "Requirement already satisfied: docopt>=0.6.2 in /usr/local/lib/python3.7/dist-packages (from num2words->textattack[tensorflow]) (0.6.2)\n", - "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers>=3.3.0->textattack[tensorflow]) (7.1.2)\n", - "Building wheels for collected packages: gdown, mpld3, overrides, segtok, sqlitedict, ftfy, langdetect, lru-dict, terminaltables, wikipedia-api, word2number\n", - " Building wheel for gdown (PEP 517) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for gdown: filename=gdown-3.12.2-py3-none-any.whl size=9704 sha256=be76d15e41eb103e46e0ba0d72cc277d90b7699bae860782279f5ee6ed86247b\n", - " Stored in directory: /root/.cache/pip/wheels/ba/e0/7e/726e872a53f7358b4b96a9975b04e98113b005cd8609a63abc\n", - " Building wheel for mpld3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for mpld3: filename=mpld3-0.3-py3-none-any.whl size=116702 sha256=729d5e596c97fd6b6855617ae575e2f299e4c3883055d82c82cd0e9f6dceebb2\n", - " Stored in directory: /root/.cache/pip/wheels/26/70/6a/1c79e59951a41b4045497da187b2724f5659ca64033cf4548e\n", - " Building wheel for overrides (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for overrides: filename=overrides-3.1.0-py3-none-any.whl size=10186 sha256=7dfa089a9708c2250125f86ed7e62bb443bdbf3d555ba72acf5e94c175dbdde6\n", - " Stored in directory: /root/.cache/pip/wheels/3a/0d/38/01a9bc6e20dcfaf0a6a7b552d03137558ba1c38aea47644682\n", - " Building wheel for segtok (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for segtok: filename=segtok-1.5.10-py3-none-any.whl size=25030 sha256=e77700c3b756745ae1e92f298016bf2f0b539bdfd6b5a90ee918e18139587df2\n", - " Stored in directory: /root/.cache/pip/wheels/67/b7/d0/a121106e61339eee5ed083bc230b1c8dc422c49a5a28c2addd\n", - " Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for sqlitedict: filename=sqlitedict-1.7.0-py3-none-any.whl size=14392 sha256=259e857c9f3cfc72c0c8b50d931b88b79d4f5382b8a70275d203a89427ffbb14\n", - " Stored in directory: /root/.cache/pip/wheels/af/94/06/18c0e83e9e227da8f3582810b51f319bbfd181e508676a56c8\n", - " Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for ftfy: filename=ftfy-6.0.3-py3-none-any.whl size=41933 sha256=c66614666a8be4942d16fab87ee11128fc03fa8424fc0a12c0f3600801fa687e\n", - " Stored in directory: /root/.cache/pip/wheels/19/f5/38/273eb3b5e76dfd850619312f693716ac4518b498f5ffb6f56d\n", - " Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993242 sha256=553fa791a31538b8828322bde06203789fc30bbbdf4bb0c80a7d7003632aa0ae\n", - " Stored in directory: /root/.cache/pip/wheels/c5/96/8a/f90c59ed25d75e50a8c10a1b1c2d4c402e4dacfa87f3aff36a\n", - " Building wheel for lru-dict (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for lru-dict: filename=lru_dict-1.1.7-cp37-cp37m-linux_x86_64.whl size=28412 sha256=aef5a191c8f01db62a275cc05e3befaee153aaaabd41f41aa2d724b4678b60ec\n", - " Stored in directory: /root/.cache/pip/wheels/9d/0b/4e/aa8fec9833090cd52bcd76f92f9d95e1ee7b915c12093663b4\n", - " Building wheel for terminaltables (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for terminaltables: filename=terminaltables-3.1.0-py3-none-any.whl size=15354 sha256=03639ca3ebb53caeaa19ad643c2149cb1884638f368b72fd18e53b90a7d52d1b\n", - " Stored in directory: /root/.cache/pip/wheels/ba/ad/c8/2d98360791161cd3db6daf6b5e730f34021fc9367d5879f497\n", - " Building wheel for wikipedia-api (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for wikipedia-api: filename=Wikipedia_API-0.5.4-py3-none-any.whl size=13475 sha256=a8fb9efb0f94278251a1983fbd3d07e4fff610ef8ea1205bb2433a4866b79b15\n", - " Stored in directory: /root/.cache/pip/wheels/d3/24/56/58ba93cf78be162451144e7a9889603f437976ef1ae7013d04\n", - " Building wheel for word2number (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for word2number: filename=word2number-1.1-py3-none-any.whl size=5580 sha256=cee3a246b5d687cb221e19e789bdcc409ea1f97d9c59d66a6a205c0f3c62457d\n", - " Stored in directory: /root/.cache/pip/wheels/4b/c3/77/a5f48aeb0d3efb7cd5ad61cbd3da30bbf9ffc9662b07c9f879\n", - "Successfully built gdown mpld3 overrides segtok sqlitedict ftfy langdetect lru-dict terminaltables wikipedia-api word2number\n", - "Installing collected packages: requests, tqdm, pyyaml, importlib-metadata, tokenizers, sentencepiece, sacremoses, overrides, huggingface-hub, xxhash, wikipedia-api, transformers, sqlitedict, segtok, mpld3, more-itertools, langdetect, konoha, janome, gdown, ftfy, fsspec, deprecated, conllu, bpemb, word2number, terminaltables, num2words, lru-dict, lemminflect, language-tool-python, flair, datasets, bert-score, textattack, tensorflow-text, tensorboardX\n", - " Attempting uninstall: requests\n", - " Found existing installation: requests 2.23.0\n", - " Uninstalling requests-2.23.0:\n", - " Successfully uninstalled requests-2.23.0\n", - " Attempting uninstall: tqdm\n", - " Found existing installation: tqdm 4.62.3\n", - " Uninstalling tqdm-4.62.3:\n", - " Successfully uninstalled tqdm-4.62.3\n", - " Attempting uninstall: pyyaml\n", - " Found existing installation: PyYAML 3.13\n", - " Uninstalling PyYAML-3.13:\n", - " Successfully uninstalled PyYAML-3.13\n", - " Attempting uninstall: importlib-metadata\n", - " Found existing installation: importlib-metadata 4.8.1\n", - " Uninstalling importlib-metadata-4.8.1:\n", - " Successfully uninstalled importlib-metadata-4.8.1\n", - " Attempting uninstall: more-itertools\n", - " Found existing installation: more-itertools 8.10.0\n", - " Uninstalling more-itertools-8.10.0:\n", - " Successfully uninstalled more-itertools-8.10.0\n", - " Attempting uninstall: gdown\n", - " Found existing installation: gdown 3.6.4\n", - " Uninstalling gdown-3.6.4:\n", - " Successfully uninstalled gdown-3.6.4\n", - "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.26.0 which is incompatible.\n", - "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n", - "Successfully installed bert-score-0.3.10 bpemb-0.3.3 conllu-4.4.1 datasets-1.11.0 deprecated-1.2.13 flair-0.9 fsspec-2021.10.0 ftfy-6.0.3 gdown-3.12.2 huggingface-hub-0.0.19 importlib-metadata-3.10.1 janome-0.4.1 konoha-4.6.5 langdetect-1.0.9 language-tool-python-2.6.1 lemminflect-0.2.2 lru-dict-1.1.7 more-itertools-8.8.0 mpld3-0.3 num2words-0.5.10 overrides-3.1.0 pyyaml-5.4.1 requests-2.26.0 sacremoses-0.0.46 segtok-1.5.10 sentencepiece-0.1.95 sqlitedict-1.7.0 tensorboardX-2.4 tensorflow-text-2.6.0 terminaltables-3.1.0 textattack-0.3.3 tokenizers-0.10.3 tqdm-4.49.0 transformers-4.11.3 wikipedia-api-0.5.4 word2number-1.1 xxhash-2.0.2\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ONayD5EJseoG" - }, - "source": [ - "## Training\n", - "\n", - "First, we're going to train a model. TextAttack integrates directly with [transformers](https://github.com/huggingface/transformers/) and [datasets](https://github.com/huggingface/datasets) to train any of the `transformers` pre-trained models on datasets from `datasets`. \n", - "\n", - "Let's use the Rotten Tomatoes Movie Review dataset: it's relatively short , and showcasesthe key features of `textattack train`. Let's take a look at the dataset using `textattack peek-dataset`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "spS2eW5WseoG", - "outputId": "795de4af-18b2-4750-d817-a03959c4cd25" - }, - "source": [ - "!textattack peek-dataset --dataset-from-huggingface rotten_tomatoes" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\u001b[34;1mtextattack\u001b[0m: Updating TextAttack package dependencies.\n", - "\u001b[34;1mtextattack\u001b[0m: Downloading NLTK required packages.\n", - "[nltk_data] Downloading package averaged_perceptron_tagger to\n", - "[nltk_data] /root/nltk_data...\n", - "[nltk_data] Unzipping taggers/averaged_perceptron_tagger.zip.\n", - "[nltk_data] Downloading package stopwords to /root/nltk_data...\n", - "[nltk_data] Unzipping corpora/stopwords.zip.\n", - "[nltk_data] Downloading package omw to /root/nltk_data...\n", - "[nltk_data] Unzipping corpora/omw.zip.\n", - "[nltk_data] Downloading package universal_tagset to /root/nltk_data...\n", - "[nltk_data] Unzipping taggers/universal_tagset.zip.\n", - "[nltk_data] Downloading package wordnet to /root/nltk_data...\n", - "[nltk_data] Unzipping corpora/wordnet.zip.\n", - "[nltk_data] Downloading package punkt to /root/nltk_data...\n", - "[nltk_data] Unzipping tokenizers/punkt.zip.\n", - "\u001b[34;1mtextattack\u001b[0m: Downloading https://textattack.s3.amazonaws.com/word_embeddings/paragramcf.\n", - "100% 481M/481M [00:39<00:00, 12.3MB/s]\n", - "\u001b[34;1mtextattack\u001b[0m: Unzipping file /root/.cache/textattack/tmpc6pdkqvf.zip to /root/.cache/textattack/word_embeddings/paragramcf.\n", - "\u001b[34;1mtextattack\u001b[0m: Successfully saved word_embeddings/paragramcf to cache.\n", - "Downloading: 5.11kB [00:00, 4.99MB/s] \n", - "Downloading: 2.02kB [00:00, 2.04MB/s] \n", - "Using custom data configuration default\n", - "Downloading and preparing dataset rotten_tomatoes_movie_review/default (download: 476.34 KiB, generated: 1.28 MiB, post-processed: Unknown size, total: 1.75 MiB) to /root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5...\n", - "Downloading: 100% 488k/488k [00:00<00:00, 28.3MB/s]\n", - "Dataset rotten_tomatoes_movie_review downloaded and prepared to /root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5. Subsequent calls will reuse this data.\n", - "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtrain\u001b[0m.\n", - "\u001b[34;1mtextattack\u001b[0m: Number of samples: \u001b[94m8530\u001b[0m\n", - "\u001b[34;1mtextattack\u001b[0m: Number of words per input:\n", - "\u001b[34;1mtextattack\u001b[0m: \ttotal: \u001b[94m157755\u001b[0m\n", - "\u001b[34;1mtextattack\u001b[0m: \tmean: \u001b[94m18.49\u001b[0m\n", - "\u001b[34;1mtextattack\u001b[0m: \tstd: \u001b[94m8.58\u001b[0m\n", - "\u001b[34;1mtextattack\u001b[0m: \tmin: \u001b[94m1\u001b[0m\n", - "\u001b[34;1mtextattack\u001b[0m: \tmax: \u001b[94m51\u001b[0m\n", - "\u001b[34;1mtextattack\u001b[0m: Dataset lowercased: \u001b[94mTrue\u001b[0m\n", - "\u001b[34;1mtextattack\u001b[0m: First sample:\n", - "the rock is destined to be the 21st century's new \" conan \" and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal . \n", - "\n", - "\u001b[34;1mtextattack\u001b[0m: Last sample:\n", - "things really get weird , though not particularly scary : the movie is all portent and no content . \n", - "\n", - "\u001b[34;1mtextattack\u001b[0m: Found 2 distinct outputs.\n", - "\u001b[34;1mtextattack\u001b[0m: Most common outputs:\n", - "\t 1 (4265)\n", - "\t 0 (4265)\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "uguqpjnLseoI" - }, - "source": [ - "The dataset looks good! It's lowercased already, so we'll make sure our model is uncased. The longest input is 51 words, so we can cap our maximum sequence length (`--model-max-length`) at 64.\n", - "\n", - "We'll train [`distilbert-base-uncased`](https://huggingface.co/transformers/model_doc/distilbert.html), since it's a relatively small model, and a good example of how we integrate with `transformers`.\n", - "\n", - "So we have our command:\n", - "\n", - "```bash\n", - "textattack train \\ # Train a model with TextAttack\n", - " --model distilbert-base-uncased \\ # Using distilbert, uncased version, from `transformers`\n", - " --dataset rotten_tomatoes \\ # On the Rotten Tomatoes dataset\n", - " --model-num-labels 3 \\ # That has 2 labels\n", - " --model-max-length 64 \\ # With a maximum sequence length of 64\n", - " --per-device-train-batch-size 128 \\ # And batch size of 128\n", - " --num-epochs 3 \\ # For 3 epochs \n", - "```\n", - "\n", - "Now let's run it (please remember to use GPU if you have access):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "BY33W9aWseoI", - "outputId": "0b0ec80a-6cec-4113-8474-b5bd78651b6c" - }, - "source": [ - "!textattack train --model-name-or-path distilbert-base-uncased --dataset rotten_tomatoes --model-num-labels 2 --model-max-length 64 --per-device-train-batch-size 128 --num-epochs 3" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\u001b[34;1mtextattack\u001b[0m: Loading transformers AutoModelForSequenceClassification: distilbert-base-uncased\n", - "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.bias', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias']\n", - "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", - "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", - "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'pre_classifier.bias', 'classifier.bias', 'pre_classifier.weight']\n", - "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", - "Using custom data configuration default\n", - "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", - "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtrain\u001b[0m.\n", - "Using custom data configuration default\n", - "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", - "Using custom data configuration default\n", - "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", - "Using custom data configuration default\n", - "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", - "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mvalidation\u001b[0m.\n", - "\u001b[34;1mtextattack\u001b[0m: Writing logs to ./outputs/2021-10-13-17-37-27-247436/train_log.txt.\n", - "\u001b[34;1mtextattack\u001b[0m: Wrote original training args to ./outputs/2021-10-13-17-37-27-247436/training_args.json.\n", - "\u001b[34;1mtextattack\u001b[0m: ***** Running training *****\n", - "\u001b[34;1mtextattack\u001b[0m: Num examples = 8530\n", - "\u001b[34;1mtextattack\u001b[0m: Num epochs = 3\n", - "\u001b[34;1mtextattack\u001b[0m: Num clean epochs = 3\n", - "\u001b[34;1mtextattack\u001b[0m: Instantaneous batch size per device = 128\n", - "\u001b[34;1mtextattack\u001b[0m: Total train batch size (w. parallel, distributed & accumulation) = 128\n", - "\u001b[34;1mtextattack\u001b[0m: Gradient accumulation steps = 1\n", - "\u001b[34;1mtextattack\u001b[0m: Total optimization steps = 201\n", - "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n", - "\u001b[34;1mtextattack\u001b[0m: Epoch 1\n", - "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 1/3\n", - "Loss 0.68924: 100% 67/67 [01:16<00:00, 1.14s/it]\n", - "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 52.86%\n", - "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 70.83%\n", - "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/2021-10-13-17-37-27-247436/best_model/\n", - "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n", - "\u001b[34;1mtextattack\u001b[0m: Epoch 2\n", - "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 2/3\n", - "Loss 0.59931: 100% 67/67 [01:16<00:00, 1.13s/it]\n", - "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 77.07%\n", - "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 82.83%\n", - "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/2021-10-13-17-37-27-247436/best_model/\n", - "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n", - "\u001b[34;1mtextattack\u001b[0m: Epoch 3\n", - "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 3/3\n", - "Loss 0.51365: 100% 67/67 [01:16<00:00, 1.14s/it]\n", - "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 85.67%\n", - "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 84.80%\n", - "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/2021-10-13-17-37-27-247436/best_model/\n", - "\u001b[34;1mtextattack\u001b[0m: Wrote README to ./outputs/2021-10-13-17-37-27-247436/README.md.\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4xzv3BGLseoI" - }, - "source": [ - "## Evaluation\n", - "\n", - "We successfully fine-tuned `distilbert-base-cased` for 3 epochs. Now let's evaluate it using `textattack eval`. This is as simple as providing the path to the pretrained model (that you just obtain from running the above command!) to `--model`, along with the number of evaluation samples. `textattack eval` will automatically load the evaluation data from training:" - ] + { + "cell_type": "markdown", + "metadata": { + "id": "pGv59SZzseoG" + }, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/QData/TextAttack/blob/master/docs/2notebook/0_End_to_End.ipynb)\n", + "\n", + "[![View Source on GitHub](https://img.shields.io/badge/github-view%20source-black.svg)](https://github.com/QData/TextAttack/blob/master/docs/2notebook/0_End_to_End.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "AQTkpf9RslEA", + "outputId": "20172655-e6ce-46c0-ba57-cb3a7f955ee3" + }, + "outputs": [ { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "qGYR_W6DseoJ", - "outputId": "a4edf6d3-9ac5-4513-ea26-754b409d5847" - }, - "source": [ - "!textattack eval --num-examples 1000 --model ./outputs/2021-10-13-17-37-27-247436/best_model/ --dataset-from-huggingface rotten_tomatoes --dataset-split test" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Using custom data configuration default\n", - "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", - "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtest\u001b[0m.\n", - "\u001b[34;1mtextattack\u001b[0m: Got 1000 predictions.\n", - "\u001b[34;1mtextattack\u001b[0m: Correct 847/1000 (\u001b[94m84.70%\u001b[0m)\n" - ] - } - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting textattack[tensorflow]\n", + " Downloading textattack-0.3.3-py3-none-any.whl (361 kB)\n", + "\u001b[K |████████████████████████████████| 361 kB 4.1 MB/s \n", + "\u001b[?25hCollecting language-tool-python\n", + " Downloading language_tool_python-2.6.1-py3-none-any.whl (30 kB)\n", + "Collecting lemminflect\n", + " Downloading lemminflect-0.2.2-py3-none-any.whl (769 kB)\n", + "\u001b[K |████████████████████████████████| 769 kB 36.0 MB/s \n", + "\u001b[?25hCollecting word2number\n", + " Downloading word2number-1.1.zip (9.7 kB)\n", + "Collecting transformers>=3.3.0\n", + " Downloading transformers-4.11.3-py3-none-any.whl (2.9 MB)\n", + "\u001b[K |████████████████████████████████| 2.9 MB 37.9 MB/s \n", + "\u001b[?25hCollecting flair\n", + " Downloading flair-0.9-py3-none-any.whl (319 kB)\n", + "\u001b[K |████████████████████████████████| 319 kB 50.4 MB/s \n", + "\u001b[?25hRequirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.4.1)\n", + "Collecting terminaltables\n", + " Downloading terminaltables-3.1.0.tar.gz (12 kB)\n", + "Requirement already satisfied: more-itertools in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (8.10.0)\n", + "Requirement already satisfied: editdistance in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (0.5.3)\n", + "Collecting bert-score>=0.3.5\n", + " Downloading bert_score-0.3.10-py3-none-any.whl (59 kB)\n", + "\u001b[K |████████████████████████████████| 59 kB 7.1 MB/s \n", + "\u001b[?25hRequirement already satisfied: torch!=1.8,>=1.7.0 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.9.0+cu111)\n", + "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (3.2.5)\n", + "Collecting num2words\n", + " Downloading num2words-0.5.10-py3-none-any.whl (101 kB)\n", + "\u001b[K |████████████████████████████████| 101 kB 11.5 MB/s \n", + "\u001b[?25hCollecting datasets\n", + " Downloading datasets-1.13.0-py3-none-any.whl (285 kB)\n", + "\u001b[K |████████████████████████████████| 285 kB 53.3 MB/s \n", + "\u001b[?25hCollecting lru-dict\n", + " Downloading lru-dict-1.1.7.tar.gz (10 kB)\n", + "Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.1.5)\n", + "Collecting tqdm<4.50.0,>=4.27\n", + " Downloading tqdm-4.49.0-py2.py3-none-any.whl (69 kB)\n", + "\u001b[K |████████████████████████████████| 69 kB 7.5 MB/s \n", + "\u001b[?25hRequirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.7.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (3.3.0)\n", + "Requirement already satisfied: numpy>=1.19.2 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (1.19.5)\n", + "Requirement already satisfied: tensorflow>=2 in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (2.6.0)\n", + "Requirement already satisfied: tensorflow-hub in /usr/local/lib/python3.7/dist-packages (from textattack[tensorflow]) (0.12.0)\n", + "Collecting tensorboardX\n", + " Downloading tensorboardX-2.4-py2.py3-none-any.whl (124 kB)\n", + "\u001b[K |████████████████████████████████| 124 kB 45.0 MB/s \n", + "\u001b[?25hCollecting tensorflow-text>=2\n", + " Downloading tensorflow_text-2.6.0-cp37-cp37m-manylinux1_x86_64.whl (4.4 MB)\n", + "\u001b[K |████████████████████████████████| 4.4 MB 38.7 MB/s \n", + "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from bert-score>=0.3.5->textattack[tensorflow]) (3.2.2)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.7/dist-packages (from bert-score>=0.3.5->textattack[tensorflow]) (21.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from bert-score>=0.3.5->textattack[tensorflow]) (2.23.0)\n", + "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.9->bert-score>=0.3.5->textattack[tensorflow]) (2.4.7)\n", + "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.0.1->textattack[tensorflow]) (2.8.2)\n", + "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.0.1->textattack[tensorflow]) (2018.9)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas>=1.0.1->textattack[tensorflow]) (1.15.0)\n", + "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.12)\n", + "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (0.12.0)\n", + "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (0.2.0)\n", + "Requirement already satisfied: h5py~=3.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (3.1.0)\n", + "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (3.7.4.3)\n", + "Requirement already satisfied: tensorflow-estimator~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (2.6.0)\n", + "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.1.0)\n", + "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.12.1)\n", + "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.1.2)\n", + "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.6.3)\n", + "Requirement already satisfied: gast==0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (0.4.0)\n", + "Requirement already satisfied: tensorboard~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (2.6.0)\n", + "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (3.17.3)\n", + "Requirement already satisfied: clang~=5.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (5.0)\n", + "Requirement already satisfied: keras~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (2.6.0)\n", + "Requirement already satisfied: grpcio<2.0,>=1.37.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (1.41.0)\n", + "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (0.37.0)\n", + "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2->textattack[tensorflow]) (3.3.0)\n", + "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py~=3.1.0->tensorflow>=2->textattack[tensorflow]) (1.5.2)\n", + "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (1.35.0)\n", + "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (3.3.4)\n", + "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (0.4.6)\n", + "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (57.4.0)\n", + "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (0.6.1)\n", + "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (1.8.0)\n", + "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (1.0.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (4.7.2)\n", + "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (4.2.4)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (0.2.8)\n", + "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (1.3.0)\n", + "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (4.8.1)\n", + "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (0.4.8)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (1.24.3)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (2.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (2021.5.30)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (3.0.4)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (3.1.1)\n", + "Collecting sacremoses\n", + " Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n", + "\u001b[K |████████████████████████████████| 895 kB 38.8 MB/s \n", + "\u001b[?25hCollecting pyyaml>=5.1\n", + " Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n", + "\u001b[K |████████████████████████████████| 636 kB 30.6 MB/s \n", + "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers>=3.3.0->textattack[tensorflow]) (2019.12.20)\n", + "Collecting huggingface-hub>=0.0.17\n", + " Downloading huggingface_hub-0.0.19-py3-none-any.whl (56 kB)\n", + "\u001b[K |████████████████████████████████| 56 kB 5.2 MB/s \n", + "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n", + " Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n", + "\u001b[K |████████████████████████████████| 3.3 MB 39.6 MB/s \n", + "\u001b[?25hCollecting xxhash\n", + " Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n", + "\u001b[K |████████████████████████████████| 243 kB 54.2 MB/s \n", + "\u001b[?25hCollecting aiohttp\n", + " Downloading aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)\n", + "\u001b[K |████████████████████████████████| 1.3 MB 49.0 MB/s \n", + "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets->textattack[tensorflow]) (0.70.12.2)\n", + "Collecting fsspec[http]>=2021.05.0\n", + " Downloading fsspec-2021.10.0-py3-none-any.whl (125 kB)\n", + "\u001b[K |████████████████████████████████| 125 kB 44.3 MB/s \n", + "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets->textattack[tensorflow]) (0.3.4)\n", + "Collecting datasets\n", + " Downloading datasets-1.12.1-py3-none-any.whl (270 kB)\n", + "\u001b[K |████████████████████████████████| 270 kB 52.8 MB/s \n", + "\u001b[?25h Downloading datasets-1.12.0-py3-none-any.whl (269 kB)\n", + "\u001b[K |████████████████████████████████| 269 kB 51.8 MB/s \n", + "\u001b[?25h Downloading datasets-1.11.0-py3-none-any.whl (264 kB)\n", + "\u001b[K |████████████████████████████████| 264 kB 50.7 MB/s \n", + "\u001b[?25hRequirement already satisfied: pyarrow!=4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets->textattack[tensorflow]) (3.0.0)\n", + "Collecting sqlitedict>=1.6.0\n", + " Downloading sqlitedict-1.7.0.tar.gz (28 kB)\n", + "Collecting more-itertools\n", + " Downloading more_itertools-8.8.0-py3-none-any.whl (48 kB)\n", + "\u001b[K |████████████████████████████████| 48 kB 5.4 MB/s \n", + "\u001b[?25hCollecting conllu>=4.0\n", + " Downloading conllu-4.4.1-py2.py3-none-any.whl (15 kB)\n", + "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (0.22.2.post1)\n", + "Collecting segtok>=1.5.7\n", + " Downloading segtok-1.5.10.tar.gz (25 kB)\n", + "Collecting langdetect\n", + " Downloading langdetect-1.0.9.tar.gz (981 kB)\n", + "\u001b[K |████████████████████████████████| 981 kB 30.2 MB/s \n", + "\u001b[?25hRequirement already satisfied: gensim<=3.8.3,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (3.6.0)\n", + "Collecting sentencepiece==0.1.95\n", + " Downloading sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2 MB)\n", + "\u001b[K |████████████████████████████████| 1.2 MB 39.0 MB/s \n", + "\u001b[?25hCollecting gdown==3.12.2\n", + " Downloading gdown-3.12.2.tar.gz (8.2 kB)\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n", + "Collecting mpld3==0.3\n", + " Downloading mpld3-0.3.tar.gz (788 kB)\n", + "\u001b[K |████████████████████████████████| 788 kB 33.7 MB/s \n", + "\u001b[?25hCollecting bpemb>=0.3.2\n", + " Downloading bpemb-0.3.3-py3-none-any.whl (19 kB)\n", + "Collecting janome\n", + " Downloading Janome-0.4.1-py2.py3-none-any.whl (19.7 MB)\n", + "\u001b[K |████████████████████████████████| 19.7 MB 50 kB/s \n", + "\u001b[?25hRequirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (0.8.9)\n", + "Requirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (0.1.2)\n", + "Collecting ftfy\n", + " Downloading ftfy-6.0.3.tar.gz (64 kB)\n", + "\u001b[K |████████████████████████████████| 64 kB 2.9 MB/s \n", + "\u001b[?25hCollecting deprecated>=1.2.4\n", + " Downloading Deprecated-1.2.13-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting konoha<5.0.0,>=4.0.0\n", + " Downloading konoha-4.6.5-py3-none-any.whl (20 kB)\n", + "Collecting wikipedia-api\n", + " Downloading Wikipedia-API-0.5.4.tar.gz (18 kB)\n", + "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from flair->textattack[tensorflow]) (4.2.6)\n", + "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim<=3.8.3,>=3.4.0->flair->textattack[tensorflow]) (5.2.1)\n", + "Requirement already satisfied: pymongo in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair->textattack[tensorflow]) (3.12.0)\n", + "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair->textattack[tensorflow]) (0.16.0)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair->textattack[tensorflow]) (2.6.3)\n", + "Collecting importlib-metadata\n", + " Downloading importlib_metadata-3.10.1-py3-none-any.whl (14 kB)\n", + "Collecting requests\n", + " Downloading requests-2.26.0-py2.py3-none-any.whl (62 kB)\n", + "\u001b[K |████████████████████████████████| 62 kB 851 kB/s \n", + "\u001b[?25hCollecting overrides<4.0.0,>=3.0.0\n", + " Downloading overrides-3.1.0.tar.gz (11 kB)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.6->tensorflow>=2->textattack[tensorflow]) (3.6.0)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->bert-score>=0.3.5->textattack[tensorflow]) (0.10.0)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->bert-score>=0.3.5->textattack[tensorflow]) (1.3.2)\n", + "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.7/dist-packages (from requests->bert-score>=0.3.5->textattack[tensorflow]) (2.0.6)\n", + "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->flair->textattack[tensorflow]) (1.0.1)\n", + "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from ftfy->flair->textattack[tensorflow]) (0.2.5)\n", + "Requirement already satisfied: docopt>=0.6.2 in /usr/local/lib/python3.7/dist-packages (from num2words->textattack[tensorflow]) (0.6.2)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers>=3.3.0->textattack[tensorflow]) (7.1.2)\n", + "Building wheels for collected packages: gdown, mpld3, overrides, segtok, sqlitedict, ftfy, langdetect, lru-dict, terminaltables, wikipedia-api, word2number\n", + " Building wheel for gdown (PEP 517) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for gdown: filename=gdown-3.12.2-py3-none-any.whl size=9704 sha256=be76d15e41eb103e46e0ba0d72cc277d90b7699bae860782279f5ee6ed86247b\n", + " Stored in directory: /root/.cache/pip/wheels/ba/e0/7e/726e872a53f7358b4b96a9975b04e98113b005cd8609a63abc\n", + " Building wheel for mpld3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for mpld3: filename=mpld3-0.3-py3-none-any.whl size=116702 sha256=729d5e596c97fd6b6855617ae575e2f299e4c3883055d82c82cd0e9f6dceebb2\n", + " Stored in directory: /root/.cache/pip/wheels/26/70/6a/1c79e59951a41b4045497da187b2724f5659ca64033cf4548e\n", + " Building wheel for overrides (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for overrides: filename=overrides-3.1.0-py3-none-any.whl size=10186 sha256=7dfa089a9708c2250125f86ed7e62bb443bdbf3d555ba72acf5e94c175dbdde6\n", + " Stored in directory: /root/.cache/pip/wheels/3a/0d/38/01a9bc6e20dcfaf0a6a7b552d03137558ba1c38aea47644682\n", + " Building wheel for segtok (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for segtok: filename=segtok-1.5.10-py3-none-any.whl size=25030 sha256=e77700c3b756745ae1e92f298016bf2f0b539bdfd6b5a90ee918e18139587df2\n", + " Stored in directory: /root/.cache/pip/wheels/67/b7/d0/a121106e61339eee5ed083bc230b1c8dc422c49a5a28c2addd\n", + " Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sqlitedict: filename=sqlitedict-1.7.0-py3-none-any.whl size=14392 sha256=259e857c9f3cfc72c0c8b50d931b88b79d4f5382b8a70275d203a89427ffbb14\n", + " Stored in directory: /root/.cache/pip/wheels/af/94/06/18c0e83e9e227da8f3582810b51f319bbfd181e508676a56c8\n", + " Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for ftfy: filename=ftfy-6.0.3-py3-none-any.whl size=41933 sha256=c66614666a8be4942d16fab87ee11128fc03fa8424fc0a12c0f3600801fa687e\n", + " Stored in directory: /root/.cache/pip/wheels/19/f5/38/273eb3b5e76dfd850619312f693716ac4518b498f5ffb6f56d\n", + " Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993242 sha256=553fa791a31538b8828322bde06203789fc30bbbdf4bb0c80a7d7003632aa0ae\n", + " Stored in directory: /root/.cache/pip/wheels/c5/96/8a/f90c59ed25d75e50a8c10a1b1c2d4c402e4dacfa87f3aff36a\n", + " Building wheel for lru-dict (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for lru-dict: filename=lru_dict-1.1.7-cp37-cp37m-linux_x86_64.whl size=28412 sha256=aef5a191c8f01db62a275cc05e3befaee153aaaabd41f41aa2d724b4678b60ec\n", + " Stored in directory: /root/.cache/pip/wheels/9d/0b/4e/aa8fec9833090cd52bcd76f92f9d95e1ee7b915c12093663b4\n", + " Building wheel for terminaltables (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for terminaltables: filename=terminaltables-3.1.0-py3-none-any.whl size=15354 sha256=03639ca3ebb53caeaa19ad643c2149cb1884638f368b72fd18e53b90a7d52d1b\n", + " Stored in directory: /root/.cache/pip/wheels/ba/ad/c8/2d98360791161cd3db6daf6b5e730f34021fc9367d5879f497\n", + " Building wheel for wikipedia-api (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for wikipedia-api: filename=Wikipedia_API-0.5.4-py3-none-any.whl size=13475 sha256=a8fb9efb0f94278251a1983fbd3d07e4fff610ef8ea1205bb2433a4866b79b15\n", + " Stored in directory: /root/.cache/pip/wheels/d3/24/56/58ba93cf78be162451144e7a9889603f437976ef1ae7013d04\n", + " Building wheel for word2number (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for word2number: filename=word2number-1.1-py3-none-any.whl size=5580 sha256=cee3a246b5d687cb221e19e789bdcc409ea1f97d9c59d66a6a205c0f3c62457d\n", + " Stored in directory: /root/.cache/pip/wheels/4b/c3/77/a5f48aeb0d3efb7cd5ad61cbd3da30bbf9ffc9662b07c9f879\n", + "Successfully built gdown mpld3 overrides segtok sqlitedict ftfy langdetect lru-dict terminaltables wikipedia-api word2number\n", + "Installing collected packages: requests, tqdm, pyyaml, importlib-metadata, tokenizers, sentencepiece, sacremoses, overrides, huggingface-hub, xxhash, wikipedia-api, transformers, sqlitedict, segtok, mpld3, more-itertools, langdetect, konoha, janome, gdown, ftfy, fsspec, deprecated, conllu, bpemb, word2number, terminaltables, num2words, lru-dict, lemminflect, language-tool-python, flair, datasets, bert-score, textattack, tensorflow-text, tensorboardX\n", + " Attempting uninstall: requests\n", + " Found existing installation: requests 2.23.0\n", + " Uninstalling requests-2.23.0:\n", + " Successfully uninstalled requests-2.23.0\n", + " Attempting uninstall: tqdm\n", + " Found existing installation: tqdm 4.62.3\n", + " Uninstalling tqdm-4.62.3:\n", + " Successfully uninstalled tqdm-4.62.3\n", + " Attempting uninstall: pyyaml\n", + " Found existing installation: PyYAML 3.13\n", + " Uninstalling PyYAML-3.13:\n", + " Successfully uninstalled PyYAML-3.13\n", + " Attempting uninstall: importlib-metadata\n", + " Found existing installation: importlib-metadata 4.8.1\n", + " Uninstalling importlib-metadata-4.8.1:\n", + " Successfully uninstalled importlib-metadata-4.8.1\n", + " Attempting uninstall: more-itertools\n", + " Found existing installation: more-itertools 8.10.0\n", + " Uninstalling more-itertools-8.10.0:\n", + " Successfully uninstalled more-itertools-8.10.0\n", + " Attempting uninstall: gdown\n", + " Found existing installation: gdown 3.6.4\n", + " Uninstalling gdown-3.6.4:\n", + " Successfully uninstalled gdown-3.6.4\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.26.0 which is incompatible.\n", + "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n", + "Successfully installed bert-score-0.3.10 bpemb-0.3.3 conllu-4.4.1 datasets-1.11.0 deprecated-1.2.13 flair-0.9 fsspec-2021.10.0 ftfy-6.0.3 gdown-3.12.2 huggingface-hub-0.0.19 importlib-metadata-3.10.1 janome-0.4.1 konoha-4.6.5 langdetect-1.0.9 language-tool-python-2.6.1 lemminflect-0.2.2 lru-dict-1.1.7 more-itertools-8.8.0 mpld3-0.3 num2words-0.5.10 overrides-3.1.0 pyyaml-5.4.1 requests-2.26.0 sacremoses-0.0.46 segtok-1.5.10 sentencepiece-0.1.95 sqlitedict-1.7.0 tensorboardX-2.4 tensorflow-text-2.6.0 terminaltables-3.1.0 textattack-0.3.3 tokenizers-0.10.3 tqdm-4.49.0 transformers-4.11.3 wikipedia-api-0.5.4 word2number-1.1 xxhash-2.0.2\n" + ] + } + ], + "source": [ + "!pip3 install textattack[tensorflow]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ONayD5EJseoG" + }, + "source": [ + "## Training\n", + "\n", + "First, we're going to train a model. TextAttack integrates directly with [transformers](https://github.com/huggingface/transformers/) and [datasets](https://github.com/huggingface/datasets) to train any of the `transformers` pre-trained models on datasets from `datasets`. \n", + "\n", + "Let's use the Rotten Tomatoes Movie Review dataset: it's relatively short , and showcasesthe key features of `textattack train`. Let's take a look at the dataset using `textattack peek-dataset`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "spS2eW5WseoG", + "outputId": "795de4af-18b2-4750-d817-a03959c4cd25" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "sFPkCZShseoJ" - }, - "source": [ - "Awesome -- we were able to train a model up to 84.9% accuracy on the test dataset – with only a single command!" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34;1mtextattack\u001b[0m: Updating TextAttack package dependencies.\n", + "\u001b[34;1mtextattack\u001b[0m: Downloading NLTK required packages.\n", + "[nltk_data] Downloading package averaged_perceptron_tagger to\n", + "[nltk_data] /root/nltk_data...\n", + "[nltk_data] Unzipping taggers/averaged_perceptron_tagger.zip.\n", + "[nltk_data] Downloading package stopwords to /root/nltk_data...\n", + "[nltk_data] Unzipping corpora/stopwords.zip.\n", + "[nltk_data] Downloading package omw to /root/nltk_data...\n", + "[nltk_data] Unzipping corpora/omw.zip.\n", + "[nltk_data] Downloading package universal_tagset to /root/nltk_data...\n", + "[nltk_data] Unzipping taggers/universal_tagset.zip.\n", + "[nltk_data] Downloading package wordnet to /root/nltk_data...\n", + "[nltk_data] Unzipping corpora/wordnet.zip.\n", + "[nltk_data] Downloading package punkt to /root/nltk_data...\n", + "[nltk_data] Unzipping tokenizers/punkt.zip.\n", + "\u001b[34;1mtextattack\u001b[0m: Downloading https://textattack.s3.amazonaws.com/word_embeddings/paragramcf.\n", + "100% 481M/481M [00:39<00:00, 12.3MB/s]\n", + "\u001b[34;1mtextattack\u001b[0m: Unzipping file /root/.cache/textattack/tmpc6pdkqvf.zip to /root/.cache/textattack/word_embeddings/paragramcf.\n", + "\u001b[34;1mtextattack\u001b[0m: Successfully saved word_embeddings/paragramcf to cache.\n", + "Downloading: 5.11kB [00:00, 4.99MB/s] \n", + "Downloading: 2.02kB [00:00, 2.04MB/s] \n", + "Using custom data configuration default\n", + "Downloading and preparing dataset rotten_tomatoes_movie_review/default (download: 476.34 KiB, generated: 1.28 MiB, post-processed: Unknown size, total: 1.75 MiB) to /root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5...\n", + "Downloading: 100% 488k/488k [00:00<00:00, 28.3MB/s]\n", + "Dataset rotten_tomatoes_movie_review downloaded and prepared to /root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5. Subsequent calls will reuse this data.\n", + "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtrain\u001b[0m.\n", + "\u001b[34;1mtextattack\u001b[0m: Number of samples: \u001b[94m8530\u001b[0m\n", + "\u001b[34;1mtextattack\u001b[0m: Number of words per input:\n", + "\u001b[34;1mtextattack\u001b[0m: \ttotal: \u001b[94m157755\u001b[0m\n", + "\u001b[34;1mtextattack\u001b[0m: \tmean: \u001b[94m18.49\u001b[0m\n", + "\u001b[34;1mtextattack\u001b[0m: \tstd: \u001b[94m8.58\u001b[0m\n", + "\u001b[34;1mtextattack\u001b[0m: \tmin: \u001b[94m1\u001b[0m\n", + "\u001b[34;1mtextattack\u001b[0m: \tmax: \u001b[94m51\u001b[0m\n", + "\u001b[34;1mtextattack\u001b[0m: Dataset lowercased: \u001b[94mTrue\u001b[0m\n", + "\u001b[34;1mtextattack\u001b[0m: First sample:\n", + "the rock is destined to be the 21st century's new \" conan \" and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal . \n", + "\n", + "\u001b[34;1mtextattack\u001b[0m: Last sample:\n", + "things really get weird , though not particularly scary : the movie is all portent and no content . \n", + "\n", + "\u001b[34;1mtextattack\u001b[0m: Found 2 distinct outputs.\n", + "\u001b[34;1mtextattack\u001b[0m: Most common outputs:\n", + "\t 1 (4265)\n", + "\t 0 (4265)\n" + ] + } + ], + "source": [ + "!textattack peek-dataset --dataset-from-huggingface rotten_tomatoes" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uguqpjnLseoI" + }, + "source": [ + "The dataset looks good! It's lowercased already, so we'll make sure our model is uncased. The longest input is 51 words, so we can cap our maximum sequence length (`--model-max-length`) at 64.\n", + "\n", + "We'll train [`distilbert-base-uncased`](https://huggingface.co/transformers/model_doc/distilbert.html), since it's a relatively small model, and a good example of how we integrate with `transformers`.\n", + "\n", + "So we have our command:\n", + "\n", + "```bash\n", + "textattack train \\ # Train a model with TextAttack\n", + " --model distilbert-base-uncased \\ # Using distilbert, uncased version, from `transformers`\n", + " --dataset rotten_tomatoes \\ # On the Rotten Tomatoes dataset\n", + " --model-num-labels 3 \\ # That has 2 labels\n", + " --model-max-length 64 \\ # With a maximum sequence length of 64\n", + " --per-device-train-batch-size 128 \\ # And batch size of 128\n", + " --num-epochs 3 \\ # For 3 epochs \n", + "```\n", + "\n", + "Now let's run it (please remember to use GPU if you have access):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "BY33W9aWseoI", + "outputId": "0b0ec80a-6cec-4113-8474-b5bd78651b6c" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "wWglEuvUseoK" - }, - "source": [ - "## Attack\n", - "\n", - "Finally, let's attack our pre-trained model. We can do this the same way as before (by providing the path to the pretrained model to `--model`). For our attack, let's use the \"TextFooler\" attack recipe, from the paper [\"Is BERT Really Robust? A Strong Baseline for Natural Language Attack on Text Classification and Entailment\" (Jin et al, 2019)](https://arxiv.org/abs/1907.11932). We can do this by passing `--recipe textfooler` to `textattack attack`.\n", - "\n", - "> *Warning*: We're printing out 100 examples and, if the attack succeeds, their perturbations. The output of this command is going to be quite long!\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34;1mtextattack\u001b[0m: Loading transformers AutoModelForSequenceClassification: distilbert-base-uncased\n", + "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.bias', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias']\n", + "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'pre_classifier.bias', 'classifier.bias', 'pre_classifier.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + "Using custom data configuration default\n", + "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", + "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtrain\u001b[0m.\n", + "Using custom data configuration default\n", + "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", + "Using custom data configuration default\n", + "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", + "Using custom data configuration default\n", + "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", + "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mvalidation\u001b[0m.\n", + "\u001b[34;1mtextattack\u001b[0m: Writing logs to ./outputs/2021-10-13-17-37-27-247436/train_log.txt.\n", + "\u001b[34;1mtextattack\u001b[0m: Wrote original training args to ./outputs/2021-10-13-17-37-27-247436/training_args.json.\n", + "\u001b[34;1mtextattack\u001b[0m: ***** Running training *****\n", + "\u001b[34;1mtextattack\u001b[0m: Num examples = 8530\n", + "\u001b[34;1mtextattack\u001b[0m: Num epochs = 3\n", + "\u001b[34;1mtextattack\u001b[0m: Num clean epochs = 3\n", + "\u001b[34;1mtextattack\u001b[0m: Instantaneous batch size per device = 128\n", + "\u001b[34;1mtextattack\u001b[0m: Total train batch size (w. parallel, distributed & accumulation) = 128\n", + "\u001b[34;1mtextattack\u001b[0m: Gradient accumulation steps = 1\n", + "\u001b[34;1mtextattack\u001b[0m: Total optimization steps = 201\n", + "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n", + "\u001b[34;1mtextattack\u001b[0m: Epoch 1\n", + "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 1/3\n", + "Loss 0.68924: 100% 67/67 [01:16<00:00, 1.14s/it]\n", + "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 52.86%\n", + "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 70.83%\n", + "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/2021-10-13-17-37-27-247436/best_model/\n", + "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n", + "\u001b[34;1mtextattack\u001b[0m: Epoch 2\n", + "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 2/3\n", + "Loss 0.59931: 100% 67/67 [01:16<00:00, 1.13s/it]\n", + "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 77.07%\n", + "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 82.83%\n", + "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/2021-10-13-17-37-27-247436/best_model/\n", + "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n", + "\u001b[34;1mtextattack\u001b[0m: Epoch 3\n", + "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 3/3\n", + "Loss 0.51365: 100% 67/67 [01:16<00:00, 1.14s/it]\n", + "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 85.67%\n", + "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 84.80%\n", + "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/2021-10-13-17-37-27-247436/best_model/\n", + "\u001b[34;1mtextattack\u001b[0m: Wrote README to ./outputs/2021-10-13-17-37-27-247436/README.md.\n" + ] + } + ], + "source": [ + "!textattack train --model-name-or-path distilbert-base-uncased --dataset rotten_tomatoes --model-num-labels 2 --model-max-length 64 --per-device-train-batch-size 128 --num-epochs 3" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4xzv3BGLseoI" + }, + "source": [ + "## Evaluation\n", + "\n", + "We successfully fine-tuned `distilbert-base-cased` for 3 epochs. Now let's evaluate it using `textattack eval`. This is as simple as providing the path to the pretrained model (that you just obtain from running the above command!) to `--model`, along with the number of evaluation samples. `textattack eval` will automatically load the evaluation data from training:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "qGYR_W6DseoJ", + "outputId": "a4edf6d3-9ac5-4513-ea26-754b409d5847" + }, + "outputs": [ { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "vL-Bo1bgseoK", - "outputId": "aad8a4f1-bda7-4687-c79c-736201a29261" - }, - "source": [ - "!textattack attack --recipe textfooler --num-examples 100 --model ./outputs/2021-10-13-17-37-27-247436/best_model/ --dataset-from-huggingface rotten_tomatoes --dataset-split test" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Using custom data configuration default\n", - "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", - "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtest\u001b[0m.\n", - "\u001b[34;1mtextattack\u001b[0m: Unknown if model of class compatible with goal function .\n", - "Attack(\n", - " (search_method): GreedyWordSwapWIR(\n", - " (wir_method): delete\n", - " )\n", - " (goal_function): UntargetedClassification\n", - " (transformation): WordSwapEmbedding(\n", - " (max_candidates): 50\n", - " (embedding): WordEmbedding\n", - " )\n", - " (constraints): \n", - " (0): WordEmbeddingDistance(\n", - " (embedding): WordEmbedding\n", - " (min_cos_sim): 0.5\n", - " (cased): False\n", - " (include_unknown_words): True\n", - " (compare_against_original): True\n", - " )\n", - " (1): PartOfSpeech(\n", - " (tagger_type): nltk\n", - " (tagset): universal\n", - " (allow_verb_noun_swap): True\n", - " (compare_against_original): True\n", - " )\n", - " (2): UniversalSentenceEncoder(\n", - " (metric): angular\n", - " (threshold): 0.840845057\n", - " (window_size): 15\n", - " (skip_text_shorter_than_window): True\n", - " (compare_against_original): False\n", - " )\n", - " (3): RepeatModification\n", - " (4): StopwordModification\n", - " (5): InputColumnModification(\n", - " (matching_column_labels): ['premise', 'hypothesis']\n", - " (columns_to_ignore): {'premise'}\n", - " )\n", - " (is_black_box): True\n", - ") \n", - "\n", - " 0% 0/100 [00:00 device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7\n", - "2021-10-13 18:00:36.592844: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", - "2021-10-13 18:00:36.609893: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", - "2021-10-13 18:00:36.627953: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", - "2021-10-13 18:00:36.688755: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", - "2021-10-13 18:00:36.717187: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", - "2021-10-13 18:00:38.127267: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n", - " 1% 1/100 [00:22<36:37, 22.19s/it]--------------------------------------------- Result 1 ---------------------------------------------\n", - "\u001b[92mPositive (95%)\u001b[0m --> \u001b[91mNegative (61%)\u001b[0m\n", - "\n", - "lovingly photographed in the manner of a golden book sprung to life , stuart little 2 \u001b[92mmanages\u001b[0m \u001b[92msweetness\u001b[0m largely without stickiness .\n", - "\n", - "lovingly photographed in the manner of a golden book sprung to life , stuart little 2 \u001b[91madministration\u001b[0m \u001b[91mhoneyed\u001b[0m largely without stickiness .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 1 / 0 / 0 / 1: 2% 2/100 [00:22<18:20, 11.23s/it]--------------------------------------------- Result 2 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", - "\n", - "\u001b[92mconsistently\u001b[0m clever and \u001b[92msuspenseful\u001b[0m .\n", - "\n", - "\u001b[91mprogressively\u001b[0m clever and \u001b[91menigmatic\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 2 / 0 / 0 / 2: 2% 2/100 [00:22<18:20, 11.23s/it]--------------------------------------------- Result 3 ---------------------------------------------\n", - "\u001b[91mNegative (85%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "it's like a \" big chill \" reunion of the baader-meinhof gang , only these guys are more harmless pranksters than political activists .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 2 / 0 / 1 / 3: 4% 4/100 [00:22<09:10, 5.73s/it]--------------------------------------------- Result 4 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (59%)\u001b[0m\n", - "\n", - "the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur supplies with \u001b[92mtremendous\u001b[0m \u001b[92mskill\u001b[0m .\n", - "\n", - "the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur supplies with \u001b[91mstupendous\u001b[0m \u001b[91mskilful\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 3 / 0 / 1 / 4: 4% 4/100 [00:22<09:10, 5.73s/it]--------------------------------------------- Result 5 ---------------------------------------------\n", - "\u001b[91mNegative (76%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "red dragon \" never cuts corners .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 3 / 0 / 2 / 5: 6% 6/100 [00:23<06:02, 3.86s/it]--------------------------------------------- Result 6 ---------------------------------------------\n", - "\u001b[92mPositive (73%)\u001b[0m --> \u001b[91mNegative (61%)\u001b[0m\n", - "\n", - "fresnadillo has something serious to say about the \u001b[92mways\u001b[0m in which extravagant chance can distort our perspective and throw us off the path of good sense .\n", - "\n", - "fresnadillo has something serious to say about the \u001b[91mmodo\u001b[0m in which extravagant chance can distort our perspective and throw us off the path of good sense .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 4 / 0 / 2 / 6: 6% 6/100 [00:23<06:02, 3.86s/it]--------------------------------------------- Result 7 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (70%)\u001b[0m\n", - "\n", - "throws in enough clever and \u001b[92munexpected\u001b[0m \u001b[92mtwists\u001b[0m to make the formula feel fresh .\n", - "\n", - "throws in enough clever and \u001b[91munwanted\u001b[0m \u001b[91mtendrils\u001b[0m to make the formula feel fresh .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 5 / 0 / 2 / 7: 8% 8/100 [00:23<04:29, 2.93s/it]--------------------------------------------- Result 8 ---------------------------------------------\n", - "\u001b[91mNegative (81%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "weighty and ponderous but every bit as filling as the treat of the title .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 5 / 0 / 3 / 8: 8% 8/100 [00:23<04:29, 2.93s/it]--------------------------------------------- Result 9 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (72%)\u001b[0m\n", - "\n", - "a \u001b[92mreal\u001b[0m audience-pleaser that will \u001b[92mstrike\u001b[0m a \u001b[92mchord\u001b[0m with anyone who's ever waited in a doctor's office , emergency room , hospital bed or insurance company office .\n", - "\n", - "a \u001b[91mactual\u001b[0m audience-pleaser that will \u001b[91mslugged\u001b[0m a \u001b[91mchords\u001b[0m with anyone who's ever waited in a doctor's office , emergency room , hospital bed or insurance company office .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 6 / 0 / 3 / 9: 10% 10/100 [00:24<03:37, 2.41s/it]--------------------------------------------- Result 10 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (91%)\u001b[0m\n", - "\n", - "generates an \u001b[92menormous\u001b[0m feeling of empathy for its characters .\n", - "\n", - "generates an \u001b[91mdreaded\u001b[0m feeling of empathy for its characters .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 7 / 0 / 3 / 10: 10% 10/100 [00:24<03:37, 2.41s/it]--------------------------------------------- Result 11 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (53%)\u001b[0m\n", - "\n", - "exposing the ways we fool ourselves is one hour photo's real \u001b[92mstrength\u001b[0m .\n", - "\n", - "exposing the ways we fool ourselves is one hour photo's real \u001b[91mstrenght\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 8 / 0 / 3 / 11: 12% 12/100 [00:24<02:58, 2.03s/it]--------------------------------------------- Result 12 ---------------------------------------------\n", - "\u001b[91mNegative (59%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "it's up to you to decide whether to admire these people's dedication to their cause or be repelled by their dogmatism , manipulativeness and narrow , fearful view of american life .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 8 / 0 / 4 / 12: 12% 12/100 [00:24<02:58, 2.03s/it]--------------------------------------------- Result 13 ---------------------------------------------\n", - "\u001b[91mNegative (80%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "mostly , [goldbacher] just lets her complicated characters be unruly , confusing and , through it all , human .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 8 / 0 / 5 / 13: 14% 14/100 [00:24<02:32, 1.77s/it]--------------------------------------------- Result 14 ---------------------------------------------\n", - "\u001b[92mPositive (88%)\u001b[0m --> \u001b[91mNegative (96%)\u001b[0m\n", - "\n", - ". . . \u001b[92mquite\u001b[0m good at providing some \u001b[92mgood\u001b[0m old fashioned spooks .\n", - "\n", - ". . . \u001b[91mtoo\u001b[0m good at providing some \u001b[91mguten\u001b[0m old fashioned spooks .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 9 / 0 / 5 / 14: 14% 14/100 [00:24<02:32, 1.77s/it]--------------------------------------------- Result 15 ---------------------------------------------\n", - "\u001b[91mNegative (91%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "at its worst , the movie is pretty diverting ; the pity is that it rarely achieves its best .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 9 / 0 / 6 / 15: 16% 16/100 [00:25<02:14, 1.60s/it]--------------------------------------------- Result 16 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (62%)\u001b[0m\n", - "\n", - "scherfig's light-hearted \u001b[92mprofile\u001b[0m of \u001b[92memotional\u001b[0m \u001b[92mdesperation\u001b[0m is achingly \u001b[92mhonest\u001b[0m and \u001b[92mdelightfully\u001b[0m cheeky .\n", - "\n", - "scherfig's light-hearted \u001b[91mcharacterize\u001b[0m of \u001b[91mpsychiatric\u001b[0m \u001b[91mdiscouragement\u001b[0m is achingly \u001b[91mcordial\u001b[0m and \u001b[91mblithely\u001b[0m cheeky .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 10 / 0 / 6 / 16: 16% 16/100 [00:25<02:14, 1.60s/it]--------------------------------------------- Result 17 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (74%)\u001b[0m\n", - "\n", - "a \u001b[92mjourney\u001b[0m \u001b[92mspanning\u001b[0m nearly three decades of bittersweet camaraderie and history , in which we feel that we truly know what makes holly and marina tick , and our \u001b[92mhearts\u001b[0m go out to them as both continue to negotiate their \u001b[92mimperfect\u001b[0m , love-hate relationship .\n", - "\n", - "a \u001b[91mtrekking\u001b[0m \u001b[91mexpectancy\u001b[0m nearly three decades of bittersweet camaraderie and history , in which we feel that we truly know what makes holly and marina tick , and our \u001b[91mcoeur\u001b[0m go out to them as both continue to negotiate their \u001b[91minadequate\u001b[0m , love-hate relationship .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 11 / 0 / 6 / 17: 18% 18/100 [00:26<02:02, 1.50s/it]--------------------------------------------- Result 18 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (51%)\u001b[0m\n", - "\n", - "the \u001b[92mwonderfully\u001b[0m \u001b[92mlush\u001b[0m morvern callar is pure punk existentialism , and ms . ramsay and her co-writer , liana dognini , have dramatized the alan warner novel , which itself felt like an answer to irvine welsh's book trainspotting .\n", - "\n", - "the \u001b[91mappallingly\u001b[0m \u001b[91mimpeccably\u001b[0m morvern callar is pure punk existentialism , and ms . ramsay and her co-writer , liana dognini , have dramatized the alan warner novel , which itself felt like an answer to irvine welsh's book trainspotting .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 12 / 0 / 6 / 18: 18% 18/100 [00:26<02:02, 1.50s/it]--------------------------------------------- Result 19 ---------------------------------------------\n", - "\u001b[92mPositive (64%)\u001b[0m --> \u001b[91mNegative (68%)\u001b[0m\n", - "\n", - "as it \u001b[92mturns\u001b[0m out , you can go \u001b[92mhome\u001b[0m again .\n", - "\n", - "as it \u001b[91mpivot\u001b[0m out , you can go \u001b[91mhomepage\u001b[0m again .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 13 / 0 / 6 / 19: 20% 20/100 [00:27<01:49, 1.37s/it]--------------------------------------------- Result 20 ---------------------------------------------\n", - "\u001b[92mPositive (95%)\u001b[0m --> \u001b[91mNegative (78%)\u001b[0m\n", - "\n", - "you've already seen city by the sea under a variety of titles , but it's \u001b[92mworth\u001b[0m yet another visit .\n", - "\n", - "you've already seen city by the sea under a variety of titles , but it's \u001b[91mchastisement\u001b[0m yet another visit .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 14 / 0 / 6 / 20: 20% 20/100 [00:27<01:49, 1.37s/it]--------------------------------------------- Result 21 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (59%)\u001b[0m\n", - "\n", - "this kind of hands-on \u001b[92mstorytelling\u001b[0m is ultimately what \u001b[92mmakes\u001b[0m shanghai ghetto move beyond a \u001b[92mgood\u001b[0m , dry , reliable textbook and what allows it to rank with its \u001b[92mworthy\u001b[0m predecessors .\n", - "\n", - "this kind of hands-on \u001b[91mmyth\u001b[0m is ultimately what \u001b[91mdo\u001b[0m shanghai ghetto move beyond a \u001b[91mopportune\u001b[0m , dry , reliable textbook and what allows it to rank with its \u001b[91mreputable\u001b[0m predecessors .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 15 / 0 / 6 / 21: 22% 22/100 [00:28<01:42, 1.31s/it]--------------------------------------------- Result 22 ---------------------------------------------\n", - "\u001b[92mPositive (88%)\u001b[0m --> \u001b[91mNegative (56%)\u001b[0m\n", - "\n", - "making such a tragedy the backdrop to a love story risks trivializing it , though chouraqui no \u001b[92mdoubt\u001b[0m intended the \u001b[92mfilm\u001b[0m to affirm love's power to \u001b[92mhelp\u001b[0m people endure almost unimaginable horror .\n", - "\n", - "making such a tragedy the backdrop to a love story risks trivializing it , though chouraqui no \u001b[91msuspecting\u001b[0m intended the \u001b[91mmovies\u001b[0m to affirm love's power to \u001b[91mpomoc\u001b[0m people endure almost unimaginable horror .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 16 / 0 / 6 / 22: 22% 22/100 [00:28<01:42, 1.31s/it]--------------------------------------------- Result 23 ---------------------------------------------\n", - "\u001b[91mNegative (54%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "grown-up quibbles are beside the point here . the little girls understand , and mccracken knows that's all that matters .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 16 / 0 / 7 / 23: 24% 24/100 [00:29<01:33, 1.22s/it]--------------------------------------------- Result 24 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (80%)\u001b[0m\n", - "\n", - "a \u001b[92mpowerful\u001b[0m , \u001b[92mchilling\u001b[0m , and affecting \u001b[92mstudy\u001b[0m of one man's dying fall .\n", - "\n", - "a \u001b[91mconclusive\u001b[0m , \u001b[91mmacabre\u001b[0m , and affecting \u001b[91mscrutinized\u001b[0m of one man's dying fall .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 17 / 0 / 7 / 24: 24% 24/100 [00:29<01:33, 1.22s/it]--------------------------------------------- Result 25 ---------------------------------------------\n", - "\u001b[92mPositive (52%)\u001b[0m --> \u001b[91mNegative (65%)\u001b[0m\n", - "\n", - "this is a \u001b[92mfascinating\u001b[0m film because there is no clear-cut hero and no all-out villain .\n", - "\n", - "this is a \u001b[91minteresting\u001b[0m film because there is no clear-cut hero and no all-out villain .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 18 / 0 / 7 / 25: 26% 26/100 [00:29<01:24, 1.15s/it]--------------------------------------------- Result 26 ---------------------------------------------\n", - "\u001b[92mPositive (86%)\u001b[0m --> \u001b[91mNegative (89%)\u001b[0m\n", - "\n", - "a dreadful day in irish history is given \u001b[92mpassionate\u001b[0m , if somewhat flawed , treatment .\n", - "\n", - "a dreadful day in irish history is given \u001b[91mvoracious\u001b[0m , if somewhat flawed , treatment .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 19 / 0 / 7 / 26: 26% 26/100 [00:29<01:24, 1.15s/it]--------------------------------------------- Result 27 ---------------------------------------------\n", - "\u001b[92mPositive (94%)\u001b[0m --> \u001b[91mNegative (89%)\u001b[0m\n", - "\n", - ". . . a \u001b[92mgood\u001b[0m film that must have baffled the folks in the marketing department .\n", - "\n", - ". . . a \u001b[91madvisable\u001b[0m film that must have baffled the folks in the marketing department .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 20 / 0 / 7 / 27: 28% 28/100 [00:30<01:18, 1.10s/it]--------------------------------------------- Result 28 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (64%)\u001b[0m\n", - "\n", - ". . . is \u001b[92mfunny\u001b[0m in the \u001b[92mway\u001b[0m that makes you ache with sadness ( the way chekhov is funny ) , \u001b[92mprofound\u001b[0m without ever being self-important , \u001b[92mwarm\u001b[0m without ever \u001b[92msuccumbing\u001b[0m to sentimentality .\n", - "\n", - ". . . is \u001b[91moutlandish\u001b[0m in the \u001b[91mitineraries\u001b[0m that makes you ache with sadness ( the way chekhov is funny ) , \u001b[91mshum\u001b[0m without ever being self-important , \u001b[91mwarmest\u001b[0m without ever \u001b[91mfending\u001b[0m to sentimentality .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 21 / 0 / 7 / 28: 28% 28/100 [00:30<01:18, 1.10s/it]--------------------------------------------- Result 29 ---------------------------------------------\n", - "\u001b[91mNegative (94%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "devotees of star trek ii : the wrath of khan will feel a nagging sense of deja vu , and the grandeur of the best next generation episodes is lacking .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 21 / 0 / 8 / 29: 30% 30/100 [00:32<01:15, 1.07s/it]--------------------------------------------- Result 30 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91m[FAILED]\u001b[0m\n", - "\n", - "a soul-stirring documentary about the israeli/palestinian conflict as revealed through the eyes of some children who remain curious about each other against all odds .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 21 / 1 / 8 / 30: 30% 30/100 [00:32<01:15, 1.07s/it]--------------------------------------------- Result 31 ---------------------------------------------\n", - "\u001b[92mPositive (64%)\u001b[0m --> \u001b[91mNegative (87%)\u001b[0m\n", - "\n", - "what's so \u001b[92mstriking\u001b[0m about jolie's performance is that she never lets her character become a caricature -- not even with that radioactive hair .\n", - "\n", - "what's so \u001b[91mstaggering\u001b[0m about jolie's performance is that she never lets her character become a caricature -- not even with that radioactive hair .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 22 / 1 / 8 / 31: 32% 32/100 [00:32<01:08, 1.01s/it]--------------------------------------------- Result 32 ---------------------------------------------\n", - "\u001b[91mNegative (67%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "the main story . . . is compelling enough , but it's difficult to shrug off the annoyance of that chatty fish .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 22 / 1 / 9 / 32: 32% 32/100 [00:32<01:08, 1.01s/it]--------------------------------------------- Result 33 ---------------------------------------------\n", - "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (90%)\u001b[0m\n", - "\n", - "the performances are \u001b[92mimmaculate\u001b[0m , with roussillon providing comic relief .\n", - "\n", - "the performances are \u001b[91mfaultless\u001b[0m , with roussillon providing comic relief .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 23 / 1 / 9 / 33: 34% 34/100 [00:33<01:04, 1.03it/s]--------------------------------------------- Result 34 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (55%)\u001b[0m\n", - "\n", - "kinnear . . . \u001b[92mgives\u001b[0m his best screen performance with an oddly \u001b[92mwinning\u001b[0m \u001b[92mportrayal\u001b[0m of one of life's ultimate losers .\n", - "\n", - "kinnear . . . \u001b[91mstipulates\u001b[0m his best screen performance with an oddly \u001b[91mwons\u001b[0m \u001b[91msketch\u001b[0m of one of life's ultimate losers .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 24 / 1 / 9 / 34: 34% 34/100 [00:33<01:04, 1.03it/s]--------------------------------------------- Result 35 ---------------------------------------------\n", - "\u001b[91mNegative (59%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "hugh grant , who has a good line in charm , has never been more charming than in about a boy .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 24 / 1 / 10 / 35: 36% 36/100 [00:33<00:59, 1.08it/s]--------------------------------------------- Result 36 ---------------------------------------------\n", - "\u001b[92mPositive (89%)\u001b[0m --> \u001b[91mNegative (60%)\u001b[0m\n", - "\n", - "there's a lot of tooth in roger dodger . but what's \u001b[92mnice\u001b[0m is that there's a casual intelligence that permeates the script .\n", - "\n", - "there's a lot of tooth in roger dodger . but what's \u001b[91mgentil\u001b[0m is that there's a casual intelligence that permeates the script .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 25 / 1 / 10 / 36: 36% 36/100 [00:33<00:59, 1.08it/s]--------------------------------------------- Result 37 ---------------------------------------------\n", - "\u001b[91mNegative (78%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "reminiscent of alfred hitchcock's thrillers , most of the scary parts in 'signs' occur while waiting for things to happen .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 25 / 1 / 11 / 37: 38% 38/100 [00:33<00:55, 1.13it/s]--------------------------------------------- Result 38 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", - "\n", - "one of the \u001b[92mbest\u001b[0m looking and \u001b[92mstylish\u001b[0m animated movies in quite a while . . .\n", - "\n", - "one of the \u001b[91mstrictest\u001b[0m looking and \u001b[91mtrendy\u001b[0m animated movies in quite a while . . .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 26 / 1 / 11 / 38: 38% 38/100 [00:33<00:55, 1.13it/s]--------------------------------------------- Result 39 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (70%)\u001b[0m\n", - "\n", - "its use of the thriller form to examine the labyrinthine ways in which people's lives cross and change , buffeted by events seemingly out of their control , is \u001b[92mintriguing\u001b[0m , \u001b[92mprovocative\u001b[0m stuff .\n", - "\n", - "its use of the thriller form to examine the labyrinthine ways in which people's lives cross and change , buffeted by events seemingly out of their control , is \u001b[91mdisconcerting\u001b[0m , \u001b[91mincite\u001b[0m stuff .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 27 / 1 / 11 / 39: 40% 40/100 [00:34<00:51, 1.16it/s]--------------------------------------------- Result 40 ---------------------------------------------\n", - "\u001b[92mPositive (91%)\u001b[0m --> \u001b[91mNegative (54%)\u001b[0m\n", - "\n", - "denver \u001b[92mshould\u001b[0m not get the first and last look at one of the most triumphant performances of vanessa redgrave's career . it deserves to be seen everywhere .\n", - "\n", - "denver \u001b[91mwoud\u001b[0m not get the first and last look at one of the most triumphant performances of vanessa redgrave's career . it deserves to be seen everywhere .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 28 / 1 / 11 / 40: 40% 40/100 [00:34<00:51, 1.16it/s]--------------------------------------------- Result 41 ---------------------------------------------\n", - "\u001b[92mPositive (60%)\u001b[0m --> \u001b[91mNegative (64%)\u001b[0m\n", - "\n", - "you needn't be steeped in '50s sociology , pop culture or movie lore to appreciate the emotional depth of haynes' work . \u001b[92mthough\u001b[0m haynes' style apes films from the period . . . its message is not rooted in that decade .\n", - "\n", - "you needn't be steeped in '50s sociology , pop culture or movie lore to appreciate the emotional depth of haynes' work . \u001b[91malbeit\u001b[0m haynes' style apes films from the period . . . its message is not rooted in that decade .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 29 / 1 / 11 / 41: 42% 42/100 [00:34<00:48, 1.21it/s]--------------------------------------------- Result 42 ---------------------------------------------\n", - "\u001b[92mPositive (93%)\u001b[0m --> \u001b[91mNegative (59%)\u001b[0m\n", - "\n", - "waiting for godard can be \u001b[92mfruitful\u001b[0m : 'in praise of love' is the director's epitaph for himself .\n", - "\n", - "waiting for godard can be \u001b[91mpropitious\u001b[0m : 'in praise of love' is the director's epitaph for himself .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 30 / 1 / 11 / 42: 42% 42/100 [00:34<00:48, 1.20it/s]--------------------------------------------- Result 43 ---------------------------------------------\n", - "\u001b[92mPositive (94%)\u001b[0m --> \u001b[91mNegative (94%)\u001b[0m\n", - "\n", - "a gangster movie with the capacity to \u001b[92msurprise\u001b[0m .\n", - "\n", - "a gangster movie with the capacity to \u001b[91mflabbergasted\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 31 / 1 / 11 / 43: 44% 44/100 [00:35<00:44, 1.25it/s]--------------------------------------------- Result 44 ---------------------------------------------\n", - "\u001b[92mPositive (78%)\u001b[0m --> \u001b[91mNegative (82%)\u001b[0m\n", - "\n", - "the film has a laundry list of minor shortcomings , but the numerous scenes of gory mayhem are \u001b[92mworth\u001b[0m the price of admission . . . if \" gory mayhem \" is your idea of a good time .\n", - "\n", - "the film has a laundry list of minor shortcomings , but the numerous scenes of gory mayhem are \u001b[91mpriceless\u001b[0m the price of admission . . . if \" gory mayhem \" is your idea of a good time .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 32 / 1 / 11 / 44: 44% 44/100 [00:35<00:44, 1.25it/s]--------------------------------------------- Result 45 ---------------------------------------------\n", - "\u001b[91mNegative (52%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "if not a home run , then at least a solid base hit .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 32 / 1 / 12 / 45: 46% 46/100 [00:35<00:41, 1.30it/s]--------------------------------------------- Result 46 ---------------------------------------------\n", - "\u001b[92mPositive (88%)\u001b[0m --> \u001b[91mNegative (79%)\u001b[0m\n", - "\n", - "goldmember is \u001b[92mfunny\u001b[0m enough to justify the embarrassment of bringing a barf bag to the moviehouse .\n", - "\n", - "goldmember is \u001b[91mcomical\u001b[0m enough to justify the embarrassment of bringing a barf bag to the moviehouse .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 33 / 1 / 12 / 46: 46% 46/100 [00:35<00:41, 1.30it/s]--------------------------------------------- Result 47 ---------------------------------------------\n", - "\u001b[92mPositive (72%)\u001b[0m --> \u001b[91mNegative (93%)\u001b[0m\n", - "\n", - ". . . a fairly disposable yet still \u001b[92mentertaining\u001b[0m b picture .\n", - "\n", - ". . . a fairly disposable yet still \u001b[91mdroll\u001b[0m b picture .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 34 / 1 / 12 / 47: 48% 48/100 [00:36<00:39, 1.32it/s]--------------------------------------------- Result 48 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", - "\n", - "it may not be particularly \u001b[92minnovative\u001b[0m , but the film's crisp , unaffected style and air of \u001b[92mgentle\u001b[0m \u001b[92mlonging\u001b[0m make it unexpectedly \u001b[92mrewarding\u001b[0m .\n", - "\n", - "it may not be particularly \u001b[91munpublished\u001b[0m , but the film's crisp , unaffected style and air of \u001b[91msoft\u001b[0m \u001b[91mvacuuming\u001b[0m make it unexpectedly \u001b[91mbounties\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 35 / 1 / 12 / 48: 48% 48/100 [00:36<00:39, 1.32it/s]--------------------------------------------- Result 49 ---------------------------------------------\n", - "\u001b[92mPositive (94%)\u001b[0m --> \u001b[91mNegative (80%)\u001b[0m\n", - "\n", - "the film \u001b[92mtruly\u001b[0m does rescue [the funk brothers] from motown's shadows . it's about time .\n", - "\n", - "the film \u001b[91mawfully\u001b[0m does rescue [the funk brothers] from motown's shadows . it's about time .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 36 / 1 / 12 / 49: 50% 50/100 [00:37<00:37, 1.34it/s]--------------------------------------------- Result 50 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", - "\n", - "drawing on an \u001b[92mirresistible\u001b[0m , languid romanticism , byler \u001b[92mreveals\u001b[0m the \u001b[92mways\u001b[0m in which a sultry evening or a beer-fueled afternoon in the sun can inspire even the most retiring heart to venture forth .\n", - "\n", - "drawing on an \u001b[91mstupendous\u001b[0m , languid romanticism , byler \u001b[91mbetrays\u001b[0m the \u001b[91mmethodology\u001b[0m in which a sultry evening or a beer-fueled afternoon in the sun can inspire even the most retiring heart to venture forth .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 37 / 1 / 12 / 50: 50% 50/100 [00:37<00:37, 1.34it/s]--------------------------------------------- Result 51 ---------------------------------------------\n", - "\u001b[91mNegative (92%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "works because we're never sure if ohlinger's on the level or merely a dying , delusional man trying to get into the history books before he croaks .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 37 / 1 / 13 / 51: 52% 52/100 [00:37<00:34, 1.38it/s]--------------------------------------------- Result 52 ---------------------------------------------\n", - "\u001b[92mPositive (64%)\u001b[0m --> \u001b[91mNegative (68%)\u001b[0m\n", - "\n", - "[scherfig] \u001b[92mhas\u001b[0m made a movie that will leave you wondering about the characters' lives after the \u001b[92mclever\u001b[0m credits roll .\n", - "\n", - "[scherfig] \u001b[91mis\u001b[0m made a movie that will leave you wondering about the characters' lives after the \u001b[91mcleverer\u001b[0m credits roll .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 38 / 1 / 13 / 52: 52% 52/100 [00:37<00:34, 1.38it/s]--------------------------------------------- Result 53 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (51%)\u001b[0m\n", - "\n", - "a \u001b[92mheady\u001b[0m , \u001b[92mbiting\u001b[0m , be-bop ride through nighttime manhattan , a loquacious videologue of the \u001b[92mmodern\u001b[0m male and the lengths to which he'll go to weave a protective cocoon around his own ego .\n", - "\n", - "a \u001b[91mhectic\u001b[0m , \u001b[91mgnawing\u001b[0m , be-bop ride through nighttime manhattan , a loquacious videologue of the \u001b[91mupgraded\u001b[0m male and the lengths to which he'll go to weave a protective cocoon around his own ego .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 39 / 1 / 13 / 53: 54% 54/100 [00:38<00:32, 1.41it/s]--------------------------------------------- Result 54 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (54%)\u001b[0m\n", - "\n", - "skin of man gets a few cheap shocks from its kids-in-peril theatrics , but it also \u001b[92mtaps\u001b[0m into the \u001b[92mprimal\u001b[0m fears of young people trying to cope with the mysterious and brutal nature of adults .\n", - "\n", - "skin of man gets a few cheap shocks from its kids-in-peril theatrics , but it also \u001b[91mfaucets\u001b[0m into the \u001b[91mprimordial\u001b[0m fears of young people trying to cope with the mysterious and brutal nature of adults .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 40 / 1 / 13 / 54: 54% 54/100 [00:38<00:32, 1.41it/s]--------------------------------------------- Result 55 ---------------------------------------------\n", - "\u001b[92mPositive (86%)\u001b[0m --> \u001b[91mNegative (55%)\u001b[0m\n", - "\n", - "the piano teacher is not an easy film . it forces you to watch people doing unpleasant things to each other and themselves , and it maintains a \u001b[92mcool\u001b[0m distance from its material that is deliberately unsettling .\n", - "\n", - "the piano teacher is not an easy film . it forces you to watch people doing unpleasant things to each other and themselves , and it maintains a \u001b[91mcopacetic\u001b[0m distance from its material that is deliberately unsettling .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 41 / 1 / 13 / 55: 56% 56/100 [00:38<00:30, 1.45it/s]--------------------------------------------- Result 56 ---------------------------------------------\n", - "\u001b[92mPositive (95%)\u001b[0m --> \u001b[91mNegative (91%)\u001b[0m\n", - "\n", - "as \u001b[92mrefreshing\u001b[0m as a drink from a woodland stream .\n", - "\n", - "as \u001b[91mretrofit\u001b[0m as a drink from a woodland stream .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 42 / 1 / 13 / 56: 56% 56/100 [00:38<00:30, 1.45it/s]--------------------------------------------- Result 57 ---------------------------------------------\n", - "\u001b[92mPositive (55%)\u001b[0m --> \u001b[91mNegative (95%)\u001b[0m\n", - "\n", - "williams absolutely nails sy's queasy infatuation and overall \u001b[92mstrangeness\u001b[0m .\n", - "\n", - "williams absolutely nails sy's queasy infatuation and overall \u001b[91mennui\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 43 / 1 / 13 / 57: 58% 58/100 [00:39<00:28, 1.48it/s]--------------------------------------------- Result 58 ---------------------------------------------\n", - "\u001b[92mPositive (67%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", - "\n", - "can i admit xxx is as deep as a petri dish and as well-characterized as a telephone book but still say it was a guilty \u001b[92mpleasure\u001b[0m ?\n", - "\n", - "can i admit xxx is as deep as a petri dish and as well-characterized as a telephone book but still say it was a guilty \u001b[91mamusement\u001b[0m ?\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 44 / 1 / 13 / 58: 58% 58/100 [00:39<00:28, 1.48it/s]--------------------------------------------- Result 59 ---------------------------------------------\n", - "\u001b[92mPositive (85%)\u001b[0m --> \u001b[91mNegative (55%)\u001b[0m\n", - "\n", - "while it's nothing we haven't seen before from murphy , i spy is still fun and \u001b[92menjoyable\u001b[0m and so aggressively silly that it's more than a worthwhile effort .\n", - "\n", - "while it's nothing we haven't seen before from murphy , i spy is still fun and \u001b[91mcosy\u001b[0m and so aggressively silly that it's more than a worthwhile effort .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 45 / 1 / 13 / 59: 60% 60/100 [00:39<00:26, 1.52it/s]--------------------------------------------- Result 60 ---------------------------------------------\n", - "\u001b[91mNegative (73%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "by the time it ends in a rush of sequins , flashbulbs , blaring brass and back-stabbing babes , it has said plenty about how show business has infiltrated every corner of society -- and not always for the better .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 45 / 1 / 14 / 60: 60% 60/100 [00:39<00:26, 1.52it/s]--------------------------------------------- Result 61 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (75%)\u001b[0m\n", - "\n", - "an \u001b[92mintimate\u001b[0m contemplation of two marvelously messy lives .\n", - "\n", - "an \u001b[91msqueamish\u001b[0m contemplation of two marvelously messy lives .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 46 / 1 / 14 / 61: 62% 62/100 [00:39<00:24, 1.56it/s]--------------------------------------------- Result 62 ---------------------------------------------\n", - "\u001b[92mPositive (54%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", - "\n", - "rarely has skin looked as \u001b[92mbeautiful\u001b[0m , desirable , even delectable , as it does in trouble every day .\n", - "\n", - "rarely has skin looked as \u001b[91mnice\u001b[0m , desirable , even delectable , as it does in trouble every day .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 47 / 1 / 14 / 62: 62% 62/100 [00:39<00:24, 1.56it/s]--------------------------------------------- Result 63 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (60%)\u001b[0m\n", - "\n", - "this is one of those rare docs that paints a \u001b[92mgrand\u001b[0m picture of an \u001b[92mera\u001b[0m and \u001b[92mmakes\u001b[0m the \u001b[92mjourney\u001b[0m feel like a \u001b[92mparty\u001b[0m .\n", - "\n", - "this is one of those rare docs that paints a \u001b[91mhefty\u001b[0m picture of an \u001b[91meras\u001b[0m and \u001b[91mai\u001b[0m the \u001b[91mtrip\u001b[0m feel like a \u001b[91mportion\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 48 / 1 / 14 / 63: 64% 64/100 [00:40<00:22, 1.57it/s]--------------------------------------------- Result 64 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (94%)\u001b[0m\n", - "\n", - "\u001b[92mpoignant\u001b[0m if familiar story of a young person suspended between two cultures .\n", - "\n", - "\u001b[91mdisquieting\u001b[0m if familiar story of a young person suspended between two cultures .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 49 / 1 / 14 / 64: 64% 64/100 [00:40<00:22, 1.57it/s]--------------------------------------------- Result 65 ---------------------------------------------\n", - "\u001b[92mPositive (94%)\u001b[0m --> \u001b[91mNegative (93%)\u001b[0m\n", - "\n", - "a \u001b[92mmetaphor\u001b[0m for a modern-day urban china searching for its identity .\n", - "\n", - "a \u001b[91mcliché\u001b[0m for a modern-day urban china searching for its identity .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 50 / 1 / 14 / 65: 66% 66/100 [00:41<00:21, 1.60it/s]--------------------------------------------- Result 66 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (69%)\u001b[0m\n", - "\n", - "for all its brooding quality , ash wednesday is \u001b[92msuspenseful\u001b[0m and ultimately unpredictable , with a \u001b[92msterling\u001b[0m ensemble cast .\n", - "\n", - "for all its brooding quality , ash wednesday is \u001b[91mupsetting\u001b[0m and ultimately unpredictable , with a \u001b[91mstirling\u001b[0m ensemble cast .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 51 / 1 / 14 / 66: 66% 66/100 [00:41<00:21, 1.60it/s]--------------------------------------------- Result 67 ---------------------------------------------\n", - "\u001b[92mPositive (90%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", - "\n", - "an odd \u001b[92mdrama\u001b[0m set in the \u001b[92mworld\u001b[0m of lingerie models and bar dancers in the midwest that held my interest precisely because it didn't try to .\n", - "\n", - "an odd \u001b[91mcinematographic\u001b[0m set in the \u001b[91mglobo\u001b[0m of lingerie models and bar dancers in the midwest that held my interest precisely because it didn't try to .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 52 / 1 / 14 / 67: 68% 68/100 [00:41<00:19, 1.63it/s]--------------------------------------------- Result 68 ---------------------------------------------\n", - "\u001b[92mPositive (85%)\u001b[0m --> \u001b[91mNegative (67%)\u001b[0m\n", - "\n", - "the film feels uncomfortably \u001b[92mreal\u001b[0m , its language and locations bearing the unmistakable stamp of authority .\n", - "\n", - "the film feels uncomfortably \u001b[91mactual\u001b[0m , its language and locations bearing the unmistakable stamp of authority .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 53 / 1 / 14 / 68: 68% 68/100 [00:41<00:19, 1.63it/s]--------------------------------------------- Result 69 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (90%)\u001b[0m\n", - "\n", - "despite its faults , gangs \u001b[92mexcels\u001b[0m in spectacle and pacing .\n", - "\n", - "despite its faults , gangs \u001b[91moverwhelms\u001b[0m in spectacle and pacing .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 54 / 1 / 14 / 69: 70% 70/100 [00:42<00:18, 1.66it/s]--------------------------------------------- Result 70 ---------------------------------------------\n", - "\u001b[92mPositive (80%)\u001b[0m --> \u001b[91mNegative (51%)\u001b[0m\n", - "\n", - "\u001b[92mentertaining\u001b[0m despite its one-joke premise with the thesis that women from venus and men from mars can indeed get together .\n", - "\n", - "\u001b[91mamusing\u001b[0m despite its one-joke premise with the thesis that women from venus and men from mars can indeed get together .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 55 / 1 / 14 / 70: 70% 70/100 [00:42<00:18, 1.66it/s]--------------------------------------------- Result 71 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (91%)\u001b[0m\n", - "\n", - "a tightly directed , \u001b[92mhighly\u001b[0m professional film that's old-fashioned in all the best possible ways .\n", - "\n", - "a tightly directed , \u001b[91mexcessively\u001b[0m professional film that's old-fashioned in all the best possible ways .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 56 / 1 / 14 / 71: 72% 72/100 [00:43<00:16, 1.66it/s]--------------------------------------------- Result 72 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (60%)\u001b[0m\n", - "\n", - "it's \u001b[92mdark\u001b[0m but has \u001b[92mwonderfully\u001b[0m \u001b[92mfunny\u001b[0m \u001b[92mmoments\u001b[0m ; you \u001b[92mcare\u001b[0m about the characters ; and the \u001b[92maction\u001b[0m and special effects are first-rate .\n", - "\n", - "it's \u001b[91mghoulish\u001b[0m but has \u001b[91munspeakably\u001b[0m \u001b[91mjoke\u001b[0m \u001b[91mmins\u001b[0m ; you \u001b[91mzorg\u001b[0m about the characters ; and the \u001b[91moperating\u001b[0m and special effects are first-rate .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 57 / 1 / 14 / 72: 72% 72/100 [00:43<00:16, 1.66it/s]--------------------------------------------- Result 73 ---------------------------------------------\n", - "\u001b[92mPositive (89%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", - "\n", - "in visual fertility \u001b[92mtreasure\u001b[0m planet rivals the \u001b[92mtop\u001b[0m japanese animations of recent vintage .\n", - "\n", - "in visual fertility \u001b[91mcoffer\u001b[0m planet rivals the \u001b[91msupremo\u001b[0m japanese animations of recent vintage .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 58 / 1 / 14 / 73: 74% 74/100 [00:44<00:15, 1.68it/s]--------------------------------------------- Result 74 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (96%)\u001b[0m\n", - "\n", - "enormously \u001b[92menjoyable\u001b[0m , high-adrenaline \u001b[92mdocumentary\u001b[0m .\n", - "\n", - "enormously \u001b[91mdroll\u001b[0m , high-adrenaline \u001b[91mpaperwork\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 59 / 1 / 14 / 74: 74% 74/100 [00:44<00:15, 1.68it/s]--------------------------------------------- Result 75 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (50%)\u001b[0m\n", - "\n", - "buy is an accomplished actress , and this is a big , \u001b[92mjuicy\u001b[0m role .\n", - "\n", - "buy is an accomplished actress , and this is a big , \u001b[91mcrusty\u001b[0m role .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 60 / 1 / 14 / 75: 76% 76/100 [00:44<00:14, 1.69it/s]--------------------------------------------- Result 76 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", - "\n", - "it \u001b[92mworks\u001b[0m its \u001b[92mmagic\u001b[0m with such \u001b[92mexuberance\u001b[0m and passion that the film's length becomes a part of its \u001b[92mfun\u001b[0m .\n", - "\n", - "it \u001b[91mfunctioned\u001b[0m its \u001b[91mpotions\u001b[0m with such \u001b[91melation\u001b[0m and passion that the film's length becomes a part of its \u001b[91mbanter\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 61 / 1 / 14 / 76: 76% 76/100 [00:44<00:14, 1.69it/s]--------------------------------------------- Result 77 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", - "\n", - "\u001b[92mbeautifully\u001b[0m crafted and \u001b[92mbrutally\u001b[0m \u001b[92mhonest\u001b[0m , promises offers an \u001b[92munexpected\u001b[0m \u001b[92mwindow\u001b[0m into the complexities of the middle east \u001b[92mstruggle\u001b[0m and into the \u001b[92mhumanity\u001b[0m of its people .\n", - "\n", - "\u001b[91mimpossibly\u001b[0m crafted and \u001b[91mhastily\u001b[0m \u001b[91mveritable\u001b[0m , promises offers an \u001b[91munforeseen\u001b[0m \u001b[91mfibreglass\u001b[0m into the complexities of the middle east \u001b[91mtussle\u001b[0m and into the \u001b[91mhumans\u001b[0m of its people .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 62 / 1 / 14 / 77: 78% 78/100 [00:46<00:13, 1.68it/s]--------------------------------------------- Result 78 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (67%)\u001b[0m\n", - "\n", - "an old-fashioned but emotionally \u001b[92mstirring\u001b[0m adventure tale of the kind they rarely make anymore .\n", - "\n", - "an old-fashioned but emotionally \u001b[91mwavering\u001b[0m adventure tale of the kind they rarely make anymore .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 63 / 1 / 14 / 78: 78% 78/100 [00:46<00:13, 1.68it/s]--------------------------------------------- Result 79 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (93%)\u001b[0m\n", - "\n", - "charlotte sometimes is a \u001b[92mgem\u001b[0m . it's always \u001b[92menthralling\u001b[0m .\n", - "\n", - "charlotte sometimes is a \u001b[91mbling\u001b[0m . it's always \u001b[91mhallucinatory\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 64 / 1 / 14 / 79: 80% 80/100 [00:47<00:11, 1.70it/s]--------------------------------------------- Result 80 ---------------------------------------------\n", - "\u001b[92mPositive (92%)\u001b[0m --> \u001b[91mNegative (50%)\u001b[0m\n", - "\n", - "in my opinion , analyze that is not as funny or entertaining as \u001b[92manalyze\u001b[0m this , but it is a \u001b[92mrespectable\u001b[0m sequel .\n", - "\n", - "in my opinion , analyze that is not as funny or entertaining as \u001b[91mdiscusses\u001b[0m this , but it is a \u001b[91mreputable\u001b[0m sequel .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 65 / 1 / 14 / 80: 80% 80/100 [00:47<00:11, 1.70it/s]--------------------------------------------- Result 81 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (94%)\u001b[0m\n", - "\n", - "a \u001b[92mremarkable\u001b[0m film by bernard rose .\n", - "\n", - "a \u001b[91mwhopping\u001b[0m film by bernard rose .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 66 / 1 / 14 / 81: 82% 82/100 [00:47<00:10, 1.72it/s]--------------------------------------------- Result 82 ---------------------------------------------\n", - "\u001b[92mPositive (80%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", - "\n", - "zhuangzhuang creates delicate balance of style , text , and subtext that's so simple and \u001b[92mprecise\u001b[0m that anything discordant would topple the balance , but against all odds , nothing does .\n", - "\n", - "zhuangzhuang creates delicate balance of style , text , and subtext that's so simple and \u001b[91mspecify\u001b[0m that anything discordant would topple the balance , but against all odds , nothing does .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 67 / 1 / 14 / 82: 82% 82/100 [00:47<00:10, 1.72it/s]--------------------------------------------- Result 83 ---------------------------------------------\n", - "\u001b[92mPositive (84%)\u001b[0m --> \u001b[91mNegative (71%)\u001b[0m\n", - "\n", - "a much more \u001b[92msuccessful\u001b[0m translation than its most famous previous film adaptation , writer-director anthony friedman's similarly updated 1970 british production .\n", - "\n", - "a much more \u001b[91mpropitious\u001b[0m translation than its most famous previous film adaptation , writer-director anthony friedman's similarly updated 1970 british production .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 68 / 1 / 14 / 83: 84% 84/100 [00:47<00:09, 1.75it/s]--------------------------------------------- Result 84 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (56%)\u001b[0m\n", - "\n", - "an \u001b[92moriginal\u001b[0m and highly cerebral examination of the psychopathic mind\n", - "\n", - "an \u001b[91mrudimentary\u001b[0m and highly cerebral examination of the psychopathic mind\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 69 / 1 / 14 / 84: 84% 84/100 [00:47<00:09, 1.75it/s]--------------------------------------------- Result 85 ---------------------------------------------\n", - "\u001b[92mPositive (93%)\u001b[0m --> \u001b[91mNegative (83%)\u001b[0m\n", - "\n", - "michel piccoli's \u001b[92mmoving\u001b[0m performance is this films reason for being .\n", - "\n", - "michel piccoli's \u001b[91mresettled\u001b[0m performance is this films reason for being .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 70 / 1 / 14 / 85: 86% 86/100 [00:48<00:07, 1.76it/s]--------------------------------------------- Result 86 ---------------------------------------------\n", - "\u001b[92mPositive (99%)\u001b[0m --> \u001b[91mNegative (95%)\u001b[0m\n", - "\n", - "a \u001b[92mcaptivating\u001b[0m and \u001b[92mintimate\u001b[0m \u001b[92mstudy\u001b[0m about \u001b[92mdying\u001b[0m and loving . . .\n", - "\n", - "a \u001b[91mhallucinatory\u001b[0m and \u001b[91mcosy\u001b[0m \u001b[91mscrutinized\u001b[0m about \u001b[91mdecedent\u001b[0m and loving . . .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 71 / 1 / 14 / 86: 86% 86/100 [00:48<00:07, 1.76it/s]--------------------------------------------- Result 87 ---------------------------------------------\n", - "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (75%)\u001b[0m\n", - "\n", - "this is an \u001b[92melegantly\u001b[0m \u001b[92mbalanced\u001b[0m movie -- every member of the ensemble has something fascinating to do -- that doesn't reveal even a hint of artifice .\n", - "\n", - "this is an \u001b[91mprettily\u001b[0m \u001b[91mbalancing\u001b[0m movie -- every member of the ensemble has something fascinating to do -- that doesn't reveal even a hint of artifice .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 72 / 1 / 14 / 87: 88% 88/100 [00:49<00:06, 1.78it/s]--------------------------------------------- Result 88 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (61%)\u001b[0m\n", - "\n", - "[grant] goes beyond his usual fluttering and stammering and \u001b[92mcaptures\u001b[0m the \u001b[92msoul\u001b[0m of a man in pain who gradually comes to recognize it and deal with it .\n", - "\n", - "[grant] goes beyond his usual fluttering and stammering and \u001b[91mincarcerate\u001b[0m the \u001b[91mwits\u001b[0m of a man in pain who gradually comes to recognize it and deal with it .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 73 / 1 / 14 / 88: 88% 88/100 [00:49<00:06, 1.78it/s]--------------------------------------------- Result 89 ---------------------------------------------\n", - "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", - "\n", - "a high-spirited buddy \u001b[92mmovie\u001b[0m about the \u001b[92mreunion\u001b[0m of \u001b[92mberlin\u001b[0m \u001b[92manarchists\u001b[0m who \u001b[92mface\u001b[0m \u001b[92marrest\u001b[0m 15 \u001b[92myears\u001b[0m after their \u001b[92mcrime\u001b[0m .\n", - "\n", - "a high-spirited buddy \u001b[91mvideo\u001b[0m about the \u001b[91mpooled\u001b[0m of \u001b[91mgermania\u001b[0m \u001b[91manarchist\u001b[0m who \u001b[91mfacial\u001b[0m \u001b[91mintercepted\u001b[0m 15 \u001b[91molds\u001b[0m after their \u001b[91mpenal\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 74 / 1 / 14 / 89: 90% 90/100 [00:50<00:05, 1.77it/s]--------------------------------------------- Result 90 ---------------------------------------------\n", - "\u001b[91mNegative (84%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "about the best thing you could say about narc is that it's a rock-solid little genre picture . whether you like it or not is basically a matter of taste .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 74 / 1 / 15 / 90: 90% 90/100 [00:50<00:05, 1.77it/s]--------------------------------------------- Result 91 ---------------------------------------------\n", - "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (80%)\u001b[0m\n", - "\n", - "an involving , \u001b[92minspirational\u001b[0m \u001b[92mdrama\u001b[0m that sometimes falls prey to its sob-story trappings .\n", - "\n", - "an involving , \u001b[91mincentive\u001b[0m \u001b[91mcataclysmic\u001b[0m that sometimes falls prey to its sob-story trappings .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 75 / 1 / 15 / 91: 92% 92/100 [00:51<00:04, 1.79it/s]--------------------------------------------- Result 92 ---------------------------------------------\n", - "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (75%)\u001b[0m\n", - "\n", - "some of the most \u001b[92minventive\u001b[0m silliness you are likely to witness in a movie theatre for some time .\n", - "\n", - "some of the most \u001b[91mcontrivance\u001b[0m silliness you are likely to witness in a movie theatre for some time .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 76 / 1 / 15 / 92: 92% 92/100 [00:51<00:04, 1.79it/s]--------------------------------------------- Result 93 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (57%)\u001b[0m\n", - "\n", - "canadian \u001b[92mfilmmaker\u001b[0m gary burns' \u001b[92minventive\u001b[0m and mordantly \u001b[92mhumorous\u001b[0m \u001b[92mtake\u001b[0m on the soullessness of work in the \u001b[92mcity\u001b[0m .\n", - "\n", - "canadian \u001b[91mscriptwriter\u001b[0m gary burns' \u001b[91minventor\u001b[0m and mordantly \u001b[91mprank\u001b[0m \u001b[91mtakes\u001b[0m on the soullessness of work in the \u001b[91mshing\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 77 / 1 / 15 / 93: 94% 94/100 [00:52<00:03, 1.78it/s]--------------------------------------------- Result 94 ---------------------------------------------\n", - "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (65%)\u001b[0m\n", - "\n", - "a rollicking \u001b[92mride\u001b[0m , with jaw-dropping action sequences , striking villains , a \u001b[92mgorgeous\u001b[0m color palette , astounding technology , \u001b[92mstirring\u001b[0m music and a boffo last hour that leads up to a strangely sinister happy ending .\n", - "\n", - "a rollicking \u001b[91mwrinkle\u001b[0m , with jaw-dropping action sequences , striking villains , a \u001b[91mleggy\u001b[0m color palette , astounding technology , \u001b[91magitation\u001b[0m music and a boffo last hour that leads up to a strangely sinister happy ending .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 78 / 1 / 15 / 94: 94% 94/100 [00:52<00:03, 1.78it/s]--------------------------------------------- Result 95 ---------------------------------------------\n", - "\u001b[92mPositive (99%)\u001b[0m --> \u001b[91mNegative (51%)\u001b[0m\n", - "\n", - "everyone's insecure in lovely and \u001b[92mamazing\u001b[0m , a \u001b[92mpoignant\u001b[0m and wryly amusing film about mothers , daughters and their relationships .\n", - "\n", - "everyone's insecure in lovely and \u001b[91mwhopping\u001b[0m , a \u001b[91mdisquieting\u001b[0m and wryly amusing film about mothers , daughters and their relationships .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 79 / 1 / 15 / 95: 96% 96/100 [00:53<00:02, 1.80it/s]--------------------------------------------- Result 96 ---------------------------------------------\n", - "\u001b[92mPositive (52%)\u001b[0m --> \u001b[91mNegative (75%)\u001b[0m\n", - "\n", - "the closest thing to the \u001b[92mexperience\u001b[0m of space travel\n", - "\n", - "the closest thing to the \u001b[91mpilot\u001b[0m of space travel\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 80 / 1 / 15 / 96: 96% 96/100 [00:53<00:02, 1.80it/s]--------------------------------------------- Result 97 ---------------------------------------------\n", - "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (96%)\u001b[0m\n", - "\n", - "full of \u001b[92msurprises\u001b[0m .\n", - "\n", - "full of \u001b[91mstumped\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 81 / 1 / 15 / 97: 98% 98/100 [00:54<00:01, 1.81it/s]--------------------------------------------- Result 98 ---------------------------------------------\n", - "\u001b[92mPositive (90%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", - "\n", - "connoisseurs of \u001b[92mchinese\u001b[0m film will be pleased to discover that tian's meticulous \u001b[92mtalent\u001b[0m \u001b[92mhas\u001b[0m not \u001b[92mwithered\u001b[0m during his enforced \u001b[92mhiatus\u001b[0m .\n", - "\n", - "connoisseurs of \u001b[91mcantonese\u001b[0m film will be pleased to discover that tian's meticulous \u001b[91mstaffing\u001b[0m \u001b[91mis\u001b[0m not \u001b[91mbloomed\u001b[0m during his enforced \u001b[91mharford\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 82 / 1 / 15 / 98: 98% 98/100 [00:54<00:01, 1.81it/s]--------------------------------------------- Result 99 ---------------------------------------------\n", - "\u001b[92mPositive (95%)\u001b[0m --> \u001b[91mNegative (69%)\u001b[0m\n", - "\n", - "if you can push on through the slow spots , you'll be \u001b[92mrewarded\u001b[0m with some \u001b[92mfine\u001b[0m \u001b[92macting\u001b[0m .\n", - "\n", - "if you can push on through the slow spots , you'll be \u001b[91mrecompense\u001b[0m with some \u001b[91mwondrous\u001b[0m \u001b[91mbehaving\u001b[0m .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 83 / 1 / 15 / 99: 100% 100/100 [00:54<00:00, 1.84it/s]--------------------------------------------- Result 100 ---------------------------------------------\n", - "\u001b[91mNegative (50%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", - "\n", - "an unusually dry-eyed , even analytical approach to material that is generally played for maximum moisture .\n", - "\n", - "\n", - "[Succeeded / Failed / Skipped / Total] 83 / 1 / 16 / 100: 100% 100/100 [00:54<00:00, 1.84it/s]\n", - "\n", - "+-------------------------------+--------+\n", - "| Attack Results | |\n", - "+-------------------------------+--------+\n", - "| Number of successful attacks: | 83 |\n", - "| Number of failed attacks: | 1 |\n", - "| Number of skipped attacks: | 16 |\n", - "| Original accuracy: | 84.0% |\n", - "| Accuracy under attack: | 1.0% |\n", - "| Attack success rate: | 98.81% |\n", - "| Average perturbed word %: | 13.68% |\n", - "| Average num. words per input: | 18.45 |\n", - "| Avg num queries: | 80.13 |\n", - "+-------------------------------+--------+\n" - ] - } - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Using custom data configuration default\n", + "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", + "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtest\u001b[0m.\n", + "\u001b[34;1mtextattack\u001b[0m: Got 1000 predictions.\n", + "\u001b[34;1mtextattack\u001b[0m: Correct 847/1000 (\u001b[94m84.70%\u001b[0m)\n" + ] + } + ], + "source": [ + "!textattack eval --num-examples 1000 --model ./outputs/2021-10-13-17-37-27-247436/best_model/ --dataset-from-huggingface rotten_tomatoes --dataset-split test" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sFPkCZShseoJ" + }, + "source": [ + "Awesome -- we were able to train a model up to 84.9% accuracy on the test dataset – with only a single command!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wWglEuvUseoK" + }, + "source": [ + "## Attack\n", + "\n", + "Finally, let's attack our pre-trained model. We can do this the same way as before (by providing the path to the pretrained model to `--model`). For our attack, let's use the \"TextFooler\" attack recipe, from the paper [\"Is BERT Really Robust? A Strong Baseline for Natural Language Attack on Text Classification and Entailment\" (Jin et al, 2019)](https://arxiv.org/abs/1907.11932). We can do this by passing `--recipe textfooler` to `textattack attack`.\n", + "\n", + "> *Warning*: We're printing out 100 examples and, if the attack succeeds, their perturbations. The output of this command is going to be quite long!\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "vL-Bo1bgseoK", + "outputId": "aad8a4f1-bda7-4687-c79c-736201a29261" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "iyrJM3CaseoL" - }, - "source": [ - "Looks like our model was 84% successful (makes sense - same evaluation set as `textattack eval`!), meaning that TextAttack attacked the model with 84 examples (since the attack won't run if an example is originally mispredicted). The attack success rate was 98.8%, meaning that TextFooler failed to find an adversarial example only 1.2% (1 out of 84) of the time.\n", - "\n", - "\n", - "## Conclusion\n", - "\n", - "That's all, folks! We've learned how to train, evaluate, and attack a model with TextAttack, using only three commands! 😀" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Using custom data configuration default\n", + "Reusing dataset rotten_tomatoes_movie_review (/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/e06abb624abab47e1a64608fdfe65a913f5a68c66118408032644a3285208fb5)\n", + "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtest\u001b[0m.\n", + "\u001b[34;1mtextattack\u001b[0m: Unknown if model of class compatible with goal function .\n", + "Attack(\n", + " (search_method): GreedyWordSwapWIR(\n", + " (wir_method): delete\n", + " )\n", + " (goal_function): UntargetedClassification\n", + " (transformation): WordSwapEmbedding(\n", + " (max_candidates): 50\n", + " (embedding): WordEmbedding\n", + " )\n", + " (constraints): \n", + " (0): WordEmbeddingDistance(\n", + " (embedding): WordEmbedding\n", + " (min_cos_sim): 0.5\n", + " (cased): False\n", + " (include_unknown_words): True\n", + " (compare_against_original): True\n", + " )\n", + " (1): PartOfSpeech(\n", + " (tagger_type): nltk\n", + " (tagset): universal\n", + " (allow_verb_noun_swap): True\n", + " (compare_against_original): True\n", + " )\n", + " (2): UniversalSentenceEncoder(\n", + " (metric): angular\n", + " (threshold): 0.840845057\n", + " (window_size): 15\n", + " (skip_text_shorter_than_window): True\n", + " (compare_against_original): False\n", + " )\n", + " (3): RepeatModification\n", + " (4): StopwordModification\n", + " (5): InputColumnModification(\n", + " (matching_column_labels): ['premise', 'hypothesis']\n", + " (columns_to_ignore): {'premise'}\n", + " )\n", + " (is_black_box): True\n", + ") \n", + "\n", + " 0% 0/100 [00:00 device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7\n", + "2021-10-13 18:00:36.592844: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", + "2021-10-13 18:00:36.609893: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", + "2021-10-13 18:00:36.627953: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", + "2021-10-13 18:00:36.688755: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", + "2021-10-13 18:00:36.717187: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 34133760 exceeds 10% of free system memory.\n", + "2021-10-13 18:00:38.127267: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n", + " 1% 1/100 [00:22<36:37, 22.19s/it]--------------------------------------------- Result 1 ---------------------------------------------\n", + "\u001b[92mPositive (95%)\u001b[0m --> \u001b[91mNegative (61%)\u001b[0m\n", + "\n", + "lovingly photographed in the manner of a golden book sprung to life , stuart little 2 \u001b[92mmanages\u001b[0m \u001b[92msweetness\u001b[0m largely without stickiness .\n", + "\n", + "lovingly photographed in the manner of a golden book sprung to life , stuart little 2 \u001b[91madministration\u001b[0m \u001b[91mhoneyed\u001b[0m largely without stickiness .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 1 / 0 / 0 / 1: 2% 2/100 [00:22<18:20, 11.23s/it]--------------------------------------------- Result 2 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", + "\n", + "\u001b[92mconsistently\u001b[0m clever and \u001b[92msuspenseful\u001b[0m .\n", + "\n", + "\u001b[91mprogressively\u001b[0m clever and \u001b[91menigmatic\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 2 / 0 / 0 / 2: 2% 2/100 [00:22<18:20, 11.23s/it]--------------------------------------------- Result 3 ---------------------------------------------\n", + "\u001b[91mNegative (85%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "it's like a \" big chill \" reunion of the baader-meinhof gang , only these guys are more harmless pranksters than political activists .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 2 / 0 / 1 / 3: 4% 4/100 [00:22<09:10, 5.73s/it]--------------------------------------------- Result 4 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (59%)\u001b[0m\n", + "\n", + "the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur supplies with \u001b[92mtremendous\u001b[0m \u001b[92mskill\u001b[0m .\n", + "\n", + "the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur supplies with \u001b[91mstupendous\u001b[0m \u001b[91mskilful\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 3 / 0 / 1 / 4: 4% 4/100 [00:22<09:10, 5.73s/it]--------------------------------------------- Result 5 ---------------------------------------------\n", + "\u001b[91mNegative (76%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "red dragon \" never cuts corners .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 3 / 0 / 2 / 5: 6% 6/100 [00:23<06:02, 3.86s/it]--------------------------------------------- Result 6 ---------------------------------------------\n", + "\u001b[92mPositive (73%)\u001b[0m --> \u001b[91mNegative (61%)\u001b[0m\n", + "\n", + "fresnadillo has something serious to say about the \u001b[92mways\u001b[0m in which extravagant chance can distort our perspective and throw us off the path of good sense .\n", + "\n", + "fresnadillo has something serious to say about the \u001b[91mmodo\u001b[0m in which extravagant chance can distort our perspective and throw us off the path of good sense .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 4 / 0 / 2 / 6: 6% 6/100 [00:23<06:02, 3.86s/it]--------------------------------------------- Result 7 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (70%)\u001b[0m\n", + "\n", + "throws in enough clever and \u001b[92munexpected\u001b[0m \u001b[92mtwists\u001b[0m to make the formula feel fresh .\n", + "\n", + "throws in enough clever and \u001b[91munwanted\u001b[0m \u001b[91mtendrils\u001b[0m to make the formula feel fresh .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 5 / 0 / 2 / 7: 8% 8/100 [00:23<04:29, 2.93s/it]--------------------------------------------- Result 8 ---------------------------------------------\n", + "\u001b[91mNegative (81%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "weighty and ponderous but every bit as filling as the treat of the title .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 5 / 0 / 3 / 8: 8% 8/100 [00:23<04:29, 2.93s/it]--------------------------------------------- Result 9 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (72%)\u001b[0m\n", + "\n", + "a \u001b[92mreal\u001b[0m audience-pleaser that will \u001b[92mstrike\u001b[0m a \u001b[92mchord\u001b[0m with anyone who's ever waited in a doctor's office , emergency room , hospital bed or insurance company office .\n", + "\n", + "a \u001b[91mactual\u001b[0m audience-pleaser that will \u001b[91mslugged\u001b[0m a \u001b[91mchords\u001b[0m with anyone who's ever waited in a doctor's office , emergency room , hospital bed or insurance company office .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 6 / 0 / 3 / 9: 10% 10/100 [00:24<03:37, 2.41s/it]--------------------------------------------- Result 10 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (91%)\u001b[0m\n", + "\n", + "generates an \u001b[92menormous\u001b[0m feeling of empathy for its characters .\n", + "\n", + "generates an \u001b[91mdreaded\u001b[0m feeling of empathy for its characters .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 7 / 0 / 3 / 10: 10% 10/100 [00:24<03:37, 2.41s/it]--------------------------------------------- Result 11 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (53%)\u001b[0m\n", + "\n", + "exposing the ways we fool ourselves is one hour photo's real \u001b[92mstrength\u001b[0m .\n", + "\n", + "exposing the ways we fool ourselves is one hour photo's real \u001b[91mstrenght\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 8 / 0 / 3 / 11: 12% 12/100 [00:24<02:58, 2.03s/it]--------------------------------------------- Result 12 ---------------------------------------------\n", + "\u001b[91mNegative (59%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "it's up to you to decide whether to admire these people's dedication to their cause or be repelled by their dogmatism , manipulativeness and narrow , fearful view of american life .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 8 / 0 / 4 / 12: 12% 12/100 [00:24<02:58, 2.03s/it]--------------------------------------------- Result 13 ---------------------------------------------\n", + "\u001b[91mNegative (80%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "mostly , [goldbacher] just lets her complicated characters be unruly , confusing and , through it all , human .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 8 / 0 / 5 / 13: 14% 14/100 [00:24<02:32, 1.77s/it]--------------------------------------------- Result 14 ---------------------------------------------\n", + "\u001b[92mPositive (88%)\u001b[0m --> \u001b[91mNegative (96%)\u001b[0m\n", + "\n", + ". . . \u001b[92mquite\u001b[0m good at providing some \u001b[92mgood\u001b[0m old fashioned spooks .\n", + "\n", + ". . . \u001b[91mtoo\u001b[0m good at providing some \u001b[91mguten\u001b[0m old fashioned spooks .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 9 / 0 / 5 / 14: 14% 14/100 [00:24<02:32, 1.77s/it]--------------------------------------------- Result 15 ---------------------------------------------\n", + "\u001b[91mNegative (91%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "at its worst , the movie is pretty diverting ; the pity is that it rarely achieves its best .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 9 / 0 / 6 / 15: 16% 16/100 [00:25<02:14, 1.60s/it]--------------------------------------------- Result 16 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (62%)\u001b[0m\n", + "\n", + "scherfig's light-hearted \u001b[92mprofile\u001b[0m of \u001b[92memotional\u001b[0m \u001b[92mdesperation\u001b[0m is achingly \u001b[92mhonest\u001b[0m and \u001b[92mdelightfully\u001b[0m cheeky .\n", + "\n", + "scherfig's light-hearted \u001b[91mcharacterize\u001b[0m of \u001b[91mpsychiatric\u001b[0m \u001b[91mdiscouragement\u001b[0m is achingly \u001b[91mcordial\u001b[0m and \u001b[91mblithely\u001b[0m cheeky .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 10 / 0 / 6 / 16: 16% 16/100 [00:25<02:14, 1.60s/it]--------------------------------------------- Result 17 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (74%)\u001b[0m\n", + "\n", + "a \u001b[92mjourney\u001b[0m \u001b[92mspanning\u001b[0m nearly three decades of bittersweet camaraderie and history , in which we feel that we truly know what makes holly and marina tick , and our \u001b[92mhearts\u001b[0m go out to them as both continue to negotiate their \u001b[92mimperfect\u001b[0m , love-hate relationship .\n", + "\n", + "a \u001b[91mtrekking\u001b[0m \u001b[91mexpectancy\u001b[0m nearly three decades of bittersweet camaraderie and history , in which we feel that we truly know what makes holly and marina tick , and our \u001b[91mcoeur\u001b[0m go out to them as both continue to negotiate their \u001b[91minadequate\u001b[0m , love-hate relationship .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 11 / 0 / 6 / 17: 18% 18/100 [00:26<02:02, 1.50s/it]--------------------------------------------- Result 18 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (51%)\u001b[0m\n", + "\n", + "the \u001b[92mwonderfully\u001b[0m \u001b[92mlush\u001b[0m morvern callar is pure punk existentialism , and ms . ramsay and her co-writer , liana dognini , have dramatized the alan warner novel , which itself felt like an answer to irvine welsh's book trainspotting .\n", + "\n", + "the \u001b[91mappallingly\u001b[0m \u001b[91mimpeccably\u001b[0m morvern callar is pure punk existentialism , and ms . ramsay and her co-writer , liana dognini , have dramatized the alan warner novel , which itself felt like an answer to irvine welsh's book trainspotting .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 12 / 0 / 6 / 18: 18% 18/100 [00:26<02:02, 1.50s/it]--------------------------------------------- Result 19 ---------------------------------------------\n", + "\u001b[92mPositive (64%)\u001b[0m --> \u001b[91mNegative (68%)\u001b[0m\n", + "\n", + "as it \u001b[92mturns\u001b[0m out , you can go \u001b[92mhome\u001b[0m again .\n", + "\n", + "as it \u001b[91mpivot\u001b[0m out , you can go \u001b[91mhomepage\u001b[0m again .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 13 / 0 / 6 / 19: 20% 20/100 [00:27<01:49, 1.37s/it]--------------------------------------------- Result 20 ---------------------------------------------\n", + "\u001b[92mPositive (95%)\u001b[0m --> \u001b[91mNegative (78%)\u001b[0m\n", + "\n", + "you've already seen city by the sea under a variety of titles , but it's \u001b[92mworth\u001b[0m yet another visit .\n", + "\n", + "you've already seen city by the sea under a variety of titles , but it's \u001b[91mchastisement\u001b[0m yet another visit .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 14 / 0 / 6 / 20: 20% 20/100 [00:27<01:49, 1.37s/it]--------------------------------------------- Result 21 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (59%)\u001b[0m\n", + "\n", + "this kind of hands-on \u001b[92mstorytelling\u001b[0m is ultimately what \u001b[92mmakes\u001b[0m shanghai ghetto move beyond a \u001b[92mgood\u001b[0m , dry , reliable textbook and what allows it to rank with its \u001b[92mworthy\u001b[0m predecessors .\n", + "\n", + "this kind of hands-on \u001b[91mmyth\u001b[0m is ultimately what \u001b[91mdo\u001b[0m shanghai ghetto move beyond a \u001b[91mopportune\u001b[0m , dry , reliable textbook and what allows it to rank with its \u001b[91mreputable\u001b[0m predecessors .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 15 / 0 / 6 / 21: 22% 22/100 [00:28<01:42, 1.31s/it]--------------------------------------------- Result 22 ---------------------------------------------\n", + "\u001b[92mPositive (88%)\u001b[0m --> \u001b[91mNegative (56%)\u001b[0m\n", + "\n", + "making such a tragedy the backdrop to a love story risks trivializing it , though chouraqui no \u001b[92mdoubt\u001b[0m intended the \u001b[92mfilm\u001b[0m to affirm love's power to \u001b[92mhelp\u001b[0m people endure almost unimaginable horror .\n", + "\n", + "making such a tragedy the backdrop to a love story risks trivializing it , though chouraqui no \u001b[91msuspecting\u001b[0m intended the \u001b[91mmovies\u001b[0m to affirm love's power to \u001b[91mpomoc\u001b[0m people endure almost unimaginable horror .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 16 / 0 / 6 / 22: 22% 22/100 [00:28<01:42, 1.31s/it]--------------------------------------------- Result 23 ---------------------------------------------\n", + "\u001b[91mNegative (54%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "grown-up quibbles are beside the point here . the little girls understand , and mccracken knows that's all that matters .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 16 / 0 / 7 / 23: 24% 24/100 [00:29<01:33, 1.22s/it]--------------------------------------------- Result 24 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (80%)\u001b[0m\n", + "\n", + "a \u001b[92mpowerful\u001b[0m , \u001b[92mchilling\u001b[0m , and affecting \u001b[92mstudy\u001b[0m of one man's dying fall .\n", + "\n", + "a \u001b[91mconclusive\u001b[0m , \u001b[91mmacabre\u001b[0m , and affecting \u001b[91mscrutinized\u001b[0m of one man's dying fall .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 17 / 0 / 7 / 24: 24% 24/100 [00:29<01:33, 1.22s/it]--------------------------------------------- Result 25 ---------------------------------------------\n", + "\u001b[92mPositive (52%)\u001b[0m --> \u001b[91mNegative (65%)\u001b[0m\n", + "\n", + "this is a \u001b[92mfascinating\u001b[0m film because there is no clear-cut hero and no all-out villain .\n", + "\n", + "this is a \u001b[91minteresting\u001b[0m film because there is no clear-cut hero and no all-out villain .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 18 / 0 / 7 / 25: 26% 26/100 [00:29<01:24, 1.15s/it]--------------------------------------------- Result 26 ---------------------------------------------\n", + "\u001b[92mPositive (86%)\u001b[0m --> \u001b[91mNegative (89%)\u001b[0m\n", + "\n", + "a dreadful day in irish history is given \u001b[92mpassionate\u001b[0m , if somewhat flawed , treatment .\n", + "\n", + "a dreadful day in irish history is given \u001b[91mvoracious\u001b[0m , if somewhat flawed , treatment .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 19 / 0 / 7 / 26: 26% 26/100 [00:29<01:24, 1.15s/it]--------------------------------------------- Result 27 ---------------------------------------------\n", + "\u001b[92mPositive (94%)\u001b[0m --> \u001b[91mNegative (89%)\u001b[0m\n", + "\n", + ". . . a \u001b[92mgood\u001b[0m film that must have baffled the folks in the marketing department .\n", + "\n", + ". . . a \u001b[91madvisable\u001b[0m film that must have baffled the folks in the marketing department .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 20 / 0 / 7 / 27: 28% 28/100 [00:30<01:18, 1.10s/it]--------------------------------------------- Result 28 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (64%)\u001b[0m\n", + "\n", + ". . . is \u001b[92mfunny\u001b[0m in the \u001b[92mway\u001b[0m that makes you ache with sadness ( the way chekhov is funny ) , \u001b[92mprofound\u001b[0m without ever being self-important , \u001b[92mwarm\u001b[0m without ever \u001b[92msuccumbing\u001b[0m to sentimentality .\n", + "\n", + ". . . is \u001b[91moutlandish\u001b[0m in the \u001b[91mitineraries\u001b[0m that makes you ache with sadness ( the way chekhov is funny ) , \u001b[91mshum\u001b[0m without ever being self-important , \u001b[91mwarmest\u001b[0m without ever \u001b[91mfending\u001b[0m to sentimentality .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 21 / 0 / 7 / 28: 28% 28/100 [00:30<01:18, 1.10s/it]--------------------------------------------- Result 29 ---------------------------------------------\n", + "\u001b[91mNegative (94%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "devotees of star trek ii : the wrath of khan will feel a nagging sense of deja vu , and the grandeur of the best next generation episodes is lacking .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 21 / 0 / 8 / 29: 30% 30/100 [00:32<01:15, 1.07s/it]--------------------------------------------- Result 30 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91m[FAILED]\u001b[0m\n", + "\n", + "a soul-stirring documentary about the israeli/palestinian conflict as revealed through the eyes of some children who remain curious about each other against all odds .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 21 / 1 / 8 / 30: 30% 30/100 [00:32<01:15, 1.07s/it]--------------------------------------------- Result 31 ---------------------------------------------\n", + "\u001b[92mPositive (64%)\u001b[0m --> \u001b[91mNegative (87%)\u001b[0m\n", + "\n", + "what's so \u001b[92mstriking\u001b[0m about jolie's performance is that she never lets her character become a caricature -- not even with that radioactive hair .\n", + "\n", + "what's so \u001b[91mstaggering\u001b[0m about jolie's performance is that she never lets her character become a caricature -- not even with that radioactive hair .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 22 / 1 / 8 / 31: 32% 32/100 [00:32<01:08, 1.01s/it]--------------------------------------------- Result 32 ---------------------------------------------\n", + "\u001b[91mNegative (67%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "the main story . . . is compelling enough , but it's difficult to shrug off the annoyance of that chatty fish .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 22 / 1 / 9 / 32: 32% 32/100 [00:32<01:08, 1.01s/it]--------------------------------------------- Result 33 ---------------------------------------------\n", + "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (90%)\u001b[0m\n", + "\n", + "the performances are \u001b[92mimmaculate\u001b[0m , with roussillon providing comic relief .\n", + "\n", + "the performances are \u001b[91mfaultless\u001b[0m , with roussillon providing comic relief .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 23 / 1 / 9 / 33: 34% 34/100 [00:33<01:04, 1.03it/s]--------------------------------------------- Result 34 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (55%)\u001b[0m\n", + "\n", + "kinnear . . . \u001b[92mgives\u001b[0m his best screen performance with an oddly \u001b[92mwinning\u001b[0m \u001b[92mportrayal\u001b[0m of one of life's ultimate losers .\n", + "\n", + "kinnear . . . \u001b[91mstipulates\u001b[0m his best screen performance with an oddly \u001b[91mwons\u001b[0m \u001b[91msketch\u001b[0m of one of life's ultimate losers .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 24 / 1 / 9 / 34: 34% 34/100 [00:33<01:04, 1.03it/s]--------------------------------------------- Result 35 ---------------------------------------------\n", + "\u001b[91mNegative (59%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "hugh grant , who has a good line in charm , has never been more charming than in about a boy .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 24 / 1 / 10 / 35: 36% 36/100 [00:33<00:59, 1.08it/s]--------------------------------------------- Result 36 ---------------------------------------------\n", + "\u001b[92mPositive (89%)\u001b[0m --> \u001b[91mNegative (60%)\u001b[0m\n", + "\n", + "there's a lot of tooth in roger dodger . but what's \u001b[92mnice\u001b[0m is that there's a casual intelligence that permeates the script .\n", + "\n", + "there's a lot of tooth in roger dodger . but what's \u001b[91mgentil\u001b[0m is that there's a casual intelligence that permeates the script .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 25 / 1 / 10 / 36: 36% 36/100 [00:33<00:59, 1.08it/s]--------------------------------------------- Result 37 ---------------------------------------------\n", + "\u001b[91mNegative (78%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "reminiscent of alfred hitchcock's thrillers , most of the scary parts in 'signs' occur while waiting for things to happen .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 25 / 1 / 11 / 37: 38% 38/100 [00:33<00:55, 1.13it/s]--------------------------------------------- Result 38 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", + "\n", + "one of the \u001b[92mbest\u001b[0m looking and \u001b[92mstylish\u001b[0m animated movies in quite a while . . .\n", + "\n", + "one of the \u001b[91mstrictest\u001b[0m looking and \u001b[91mtrendy\u001b[0m animated movies in quite a while . . .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 26 / 1 / 11 / 38: 38% 38/100 [00:33<00:55, 1.13it/s]--------------------------------------------- Result 39 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (70%)\u001b[0m\n", + "\n", + "its use of the thriller form to examine the labyrinthine ways in which people's lives cross and change , buffeted by events seemingly out of their control , is \u001b[92mintriguing\u001b[0m , \u001b[92mprovocative\u001b[0m stuff .\n", + "\n", + "its use of the thriller form to examine the labyrinthine ways in which people's lives cross and change , buffeted by events seemingly out of their control , is \u001b[91mdisconcerting\u001b[0m , \u001b[91mincite\u001b[0m stuff .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 27 / 1 / 11 / 39: 40% 40/100 [00:34<00:51, 1.16it/s]--------------------------------------------- Result 40 ---------------------------------------------\n", + "\u001b[92mPositive (91%)\u001b[0m --> \u001b[91mNegative (54%)\u001b[0m\n", + "\n", + "denver \u001b[92mshould\u001b[0m not get the first and last look at one of the most triumphant performances of vanessa redgrave's career . it deserves to be seen everywhere .\n", + "\n", + "denver \u001b[91mwoud\u001b[0m not get the first and last look at one of the most triumphant performances of vanessa redgrave's career . it deserves to be seen everywhere .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 28 / 1 / 11 / 40: 40% 40/100 [00:34<00:51, 1.16it/s]--------------------------------------------- Result 41 ---------------------------------------------\n", + "\u001b[92mPositive (60%)\u001b[0m --> \u001b[91mNegative (64%)\u001b[0m\n", + "\n", + "you needn't be steeped in '50s sociology , pop culture or movie lore to appreciate the emotional depth of haynes' work . \u001b[92mthough\u001b[0m haynes' style apes films from the period . . . its message is not rooted in that decade .\n", + "\n", + "you needn't be steeped in '50s sociology , pop culture or movie lore to appreciate the emotional depth of haynes' work . \u001b[91malbeit\u001b[0m haynes' style apes films from the period . . . its message is not rooted in that decade .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 29 / 1 / 11 / 41: 42% 42/100 [00:34<00:48, 1.21it/s]--------------------------------------------- Result 42 ---------------------------------------------\n", + "\u001b[92mPositive (93%)\u001b[0m --> \u001b[91mNegative (59%)\u001b[0m\n", + "\n", + "waiting for godard can be \u001b[92mfruitful\u001b[0m : 'in praise of love' is the director's epitaph for himself .\n", + "\n", + "waiting for godard can be \u001b[91mpropitious\u001b[0m : 'in praise of love' is the director's epitaph for himself .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 30 / 1 / 11 / 42: 42% 42/100 [00:34<00:48, 1.20it/s]--------------------------------------------- Result 43 ---------------------------------------------\n", + "\u001b[92mPositive (94%)\u001b[0m --> \u001b[91mNegative (94%)\u001b[0m\n", + "\n", + "a gangster movie with the capacity to \u001b[92msurprise\u001b[0m .\n", + "\n", + "a gangster movie with the capacity to \u001b[91mflabbergasted\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 31 / 1 / 11 / 43: 44% 44/100 [00:35<00:44, 1.25it/s]--------------------------------------------- Result 44 ---------------------------------------------\n", + "\u001b[92mPositive (78%)\u001b[0m --> \u001b[91mNegative (82%)\u001b[0m\n", + "\n", + "the film has a laundry list of minor shortcomings , but the numerous scenes of gory mayhem are \u001b[92mworth\u001b[0m the price of admission . . . if \" gory mayhem \" is your idea of a good time .\n", + "\n", + "the film has a laundry list of minor shortcomings , but the numerous scenes of gory mayhem are \u001b[91mpriceless\u001b[0m the price of admission . . . if \" gory mayhem \" is your idea of a good time .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 32 / 1 / 11 / 44: 44% 44/100 [00:35<00:44, 1.25it/s]--------------------------------------------- Result 45 ---------------------------------------------\n", + "\u001b[91mNegative (52%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "if not a home run , then at least a solid base hit .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 32 / 1 / 12 / 45: 46% 46/100 [00:35<00:41, 1.30it/s]--------------------------------------------- Result 46 ---------------------------------------------\n", + "\u001b[92mPositive (88%)\u001b[0m --> \u001b[91mNegative (79%)\u001b[0m\n", + "\n", + "goldmember is \u001b[92mfunny\u001b[0m enough to justify the embarrassment of bringing a barf bag to the moviehouse .\n", + "\n", + "goldmember is \u001b[91mcomical\u001b[0m enough to justify the embarrassment of bringing a barf bag to the moviehouse .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 33 / 1 / 12 / 46: 46% 46/100 [00:35<00:41, 1.30it/s]--------------------------------------------- Result 47 ---------------------------------------------\n", + "\u001b[92mPositive (72%)\u001b[0m --> \u001b[91mNegative (93%)\u001b[0m\n", + "\n", + ". . . a fairly disposable yet still \u001b[92mentertaining\u001b[0m b picture .\n", + "\n", + ". . . a fairly disposable yet still \u001b[91mdroll\u001b[0m b picture .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 34 / 1 / 12 / 47: 48% 48/100 [00:36<00:39, 1.32it/s]--------------------------------------------- Result 48 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", + "\n", + "it may not be particularly \u001b[92minnovative\u001b[0m , but the film's crisp , unaffected style and air of \u001b[92mgentle\u001b[0m \u001b[92mlonging\u001b[0m make it unexpectedly \u001b[92mrewarding\u001b[0m .\n", + "\n", + "it may not be particularly \u001b[91munpublished\u001b[0m , but the film's crisp , unaffected style and air of \u001b[91msoft\u001b[0m \u001b[91mvacuuming\u001b[0m make it unexpectedly \u001b[91mbounties\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 35 / 1 / 12 / 48: 48% 48/100 [00:36<00:39, 1.32it/s]--------------------------------------------- Result 49 ---------------------------------------------\n", + "\u001b[92mPositive (94%)\u001b[0m --> \u001b[91mNegative (80%)\u001b[0m\n", + "\n", + "the film \u001b[92mtruly\u001b[0m does rescue [the funk brothers] from motown's shadows . it's about time .\n", + "\n", + "the film \u001b[91mawfully\u001b[0m does rescue [the funk brothers] from motown's shadows . it's about time .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 36 / 1 / 12 / 49: 50% 50/100 [00:37<00:37, 1.34it/s]--------------------------------------------- Result 50 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", + "\n", + "drawing on an \u001b[92mirresistible\u001b[0m , languid romanticism , byler \u001b[92mreveals\u001b[0m the \u001b[92mways\u001b[0m in which a sultry evening or a beer-fueled afternoon in the sun can inspire even the most retiring heart to venture forth .\n", + "\n", + "drawing on an \u001b[91mstupendous\u001b[0m , languid romanticism , byler \u001b[91mbetrays\u001b[0m the \u001b[91mmethodology\u001b[0m in which a sultry evening or a beer-fueled afternoon in the sun can inspire even the most retiring heart to venture forth .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 37 / 1 / 12 / 50: 50% 50/100 [00:37<00:37, 1.34it/s]--------------------------------------------- Result 51 ---------------------------------------------\n", + "\u001b[91mNegative (92%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "works because we're never sure if ohlinger's on the level or merely a dying , delusional man trying to get into the history books before he croaks .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 37 / 1 / 13 / 51: 52% 52/100 [00:37<00:34, 1.38it/s]--------------------------------------------- Result 52 ---------------------------------------------\n", + "\u001b[92mPositive (64%)\u001b[0m --> \u001b[91mNegative (68%)\u001b[0m\n", + "\n", + "[scherfig] \u001b[92mhas\u001b[0m made a movie that will leave you wondering about the characters' lives after the \u001b[92mclever\u001b[0m credits roll .\n", + "\n", + "[scherfig] \u001b[91mis\u001b[0m made a movie that will leave you wondering about the characters' lives after the \u001b[91mcleverer\u001b[0m credits roll .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 38 / 1 / 13 / 52: 52% 52/100 [00:37<00:34, 1.38it/s]--------------------------------------------- Result 53 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (51%)\u001b[0m\n", + "\n", + "a \u001b[92mheady\u001b[0m , \u001b[92mbiting\u001b[0m , be-bop ride through nighttime manhattan , a loquacious videologue of the \u001b[92mmodern\u001b[0m male and the lengths to which he'll go to weave a protective cocoon around his own ego .\n", + "\n", + "a \u001b[91mhectic\u001b[0m , \u001b[91mgnawing\u001b[0m , be-bop ride through nighttime manhattan , a loquacious videologue of the \u001b[91mupgraded\u001b[0m male and the lengths to which he'll go to weave a protective cocoon around his own ego .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 39 / 1 / 13 / 53: 54% 54/100 [00:38<00:32, 1.41it/s]--------------------------------------------- Result 54 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (54%)\u001b[0m\n", + "\n", + "skin of man gets a few cheap shocks from its kids-in-peril theatrics , but it also \u001b[92mtaps\u001b[0m into the \u001b[92mprimal\u001b[0m fears of young people trying to cope with the mysterious and brutal nature of adults .\n", + "\n", + "skin of man gets a few cheap shocks from its kids-in-peril theatrics , but it also \u001b[91mfaucets\u001b[0m into the \u001b[91mprimordial\u001b[0m fears of young people trying to cope with the mysterious and brutal nature of adults .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 40 / 1 / 13 / 54: 54% 54/100 [00:38<00:32, 1.41it/s]--------------------------------------------- Result 55 ---------------------------------------------\n", + "\u001b[92mPositive (86%)\u001b[0m --> \u001b[91mNegative (55%)\u001b[0m\n", + "\n", + "the piano teacher is not an easy film . it forces you to watch people doing unpleasant things to each other and themselves , and it maintains a \u001b[92mcool\u001b[0m distance from its material that is deliberately unsettling .\n", + "\n", + "the piano teacher is not an easy film . it forces you to watch people doing unpleasant things to each other and themselves , and it maintains a \u001b[91mcopacetic\u001b[0m distance from its material that is deliberately unsettling .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 41 / 1 / 13 / 55: 56% 56/100 [00:38<00:30, 1.45it/s]--------------------------------------------- Result 56 ---------------------------------------------\n", + "\u001b[92mPositive (95%)\u001b[0m --> \u001b[91mNegative (91%)\u001b[0m\n", + "\n", + "as \u001b[92mrefreshing\u001b[0m as a drink from a woodland stream .\n", + "\n", + "as \u001b[91mretrofit\u001b[0m as a drink from a woodland stream .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 42 / 1 / 13 / 56: 56% 56/100 [00:38<00:30, 1.45it/s]--------------------------------------------- Result 57 ---------------------------------------------\n", + "\u001b[92mPositive (55%)\u001b[0m --> \u001b[91mNegative (95%)\u001b[0m\n", + "\n", + "williams absolutely nails sy's queasy infatuation and overall \u001b[92mstrangeness\u001b[0m .\n", + "\n", + "williams absolutely nails sy's queasy infatuation and overall \u001b[91mennui\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 43 / 1 / 13 / 57: 58% 58/100 [00:39<00:28, 1.48it/s]--------------------------------------------- Result 58 ---------------------------------------------\n", + "\u001b[92mPositive (67%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", + "\n", + "can i admit xxx is as deep as a petri dish and as well-characterized as a telephone book but still say it was a guilty \u001b[92mpleasure\u001b[0m ?\n", + "\n", + "can i admit xxx is as deep as a petri dish and as well-characterized as a telephone book but still say it was a guilty \u001b[91mamusement\u001b[0m ?\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 44 / 1 / 13 / 58: 58% 58/100 [00:39<00:28, 1.48it/s]--------------------------------------------- Result 59 ---------------------------------------------\n", + "\u001b[92mPositive (85%)\u001b[0m --> \u001b[91mNegative (55%)\u001b[0m\n", + "\n", + "while it's nothing we haven't seen before from murphy , i spy is still fun and \u001b[92menjoyable\u001b[0m and so aggressively silly that it's more than a worthwhile effort .\n", + "\n", + "while it's nothing we haven't seen before from murphy , i spy is still fun and \u001b[91mcosy\u001b[0m and so aggressively silly that it's more than a worthwhile effort .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 45 / 1 / 13 / 59: 60% 60/100 [00:39<00:26, 1.52it/s]--------------------------------------------- Result 60 ---------------------------------------------\n", + "\u001b[91mNegative (73%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "by the time it ends in a rush of sequins , flashbulbs , blaring brass and back-stabbing babes , it has said plenty about how show business has infiltrated every corner of society -- and not always for the better .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 45 / 1 / 14 / 60: 60% 60/100 [00:39<00:26, 1.52it/s]--------------------------------------------- Result 61 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (75%)\u001b[0m\n", + "\n", + "an \u001b[92mintimate\u001b[0m contemplation of two marvelously messy lives .\n", + "\n", + "an \u001b[91msqueamish\u001b[0m contemplation of two marvelously messy lives .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 46 / 1 / 14 / 61: 62% 62/100 [00:39<00:24, 1.56it/s]--------------------------------------------- Result 62 ---------------------------------------------\n", + "\u001b[92mPositive (54%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", + "\n", + "rarely has skin looked as \u001b[92mbeautiful\u001b[0m , desirable , even delectable , as it does in trouble every day .\n", + "\n", + "rarely has skin looked as \u001b[91mnice\u001b[0m , desirable , even delectable , as it does in trouble every day .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 47 / 1 / 14 / 62: 62% 62/100 [00:39<00:24, 1.56it/s]--------------------------------------------- Result 63 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (60%)\u001b[0m\n", + "\n", + "this is one of those rare docs that paints a \u001b[92mgrand\u001b[0m picture of an \u001b[92mera\u001b[0m and \u001b[92mmakes\u001b[0m the \u001b[92mjourney\u001b[0m feel like a \u001b[92mparty\u001b[0m .\n", + "\n", + "this is one of those rare docs that paints a \u001b[91mhefty\u001b[0m picture of an \u001b[91meras\u001b[0m and \u001b[91mai\u001b[0m the \u001b[91mtrip\u001b[0m feel like a \u001b[91mportion\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 48 / 1 / 14 / 63: 64% 64/100 [00:40<00:22, 1.57it/s]--------------------------------------------- Result 64 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (94%)\u001b[0m\n", + "\n", + "\u001b[92mpoignant\u001b[0m if familiar story of a young person suspended between two cultures .\n", + "\n", + "\u001b[91mdisquieting\u001b[0m if familiar story of a young person suspended between two cultures .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 49 / 1 / 14 / 64: 64% 64/100 [00:40<00:22, 1.57it/s]--------------------------------------------- Result 65 ---------------------------------------------\n", + "\u001b[92mPositive (94%)\u001b[0m --> \u001b[91mNegative (93%)\u001b[0m\n", + "\n", + "a \u001b[92mmetaphor\u001b[0m for a modern-day urban china searching for its identity .\n", + "\n", + "a \u001b[91mcliché\u001b[0m for a modern-day urban china searching for its identity .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 50 / 1 / 14 / 65: 66% 66/100 [00:41<00:21, 1.60it/s]--------------------------------------------- Result 66 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (69%)\u001b[0m\n", + "\n", + "for all its brooding quality , ash wednesday is \u001b[92msuspenseful\u001b[0m and ultimately unpredictable , with a \u001b[92msterling\u001b[0m ensemble cast .\n", + "\n", + "for all its brooding quality , ash wednesday is \u001b[91mupsetting\u001b[0m and ultimately unpredictable , with a \u001b[91mstirling\u001b[0m ensemble cast .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 51 / 1 / 14 / 66: 66% 66/100 [00:41<00:21, 1.60it/s]--------------------------------------------- Result 67 ---------------------------------------------\n", + "\u001b[92mPositive (90%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", + "\n", + "an odd \u001b[92mdrama\u001b[0m set in the \u001b[92mworld\u001b[0m of lingerie models and bar dancers in the midwest that held my interest precisely because it didn't try to .\n", + "\n", + "an odd \u001b[91mcinematographic\u001b[0m set in the \u001b[91mglobo\u001b[0m of lingerie models and bar dancers in the midwest that held my interest precisely because it didn't try to .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 52 / 1 / 14 / 67: 68% 68/100 [00:41<00:19, 1.63it/s]--------------------------------------------- Result 68 ---------------------------------------------\n", + "\u001b[92mPositive (85%)\u001b[0m --> \u001b[91mNegative (67%)\u001b[0m\n", + "\n", + "the film feels uncomfortably \u001b[92mreal\u001b[0m , its language and locations bearing the unmistakable stamp of authority .\n", + "\n", + "the film feels uncomfortably \u001b[91mactual\u001b[0m , its language and locations bearing the unmistakable stamp of authority .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 53 / 1 / 14 / 68: 68% 68/100 [00:41<00:19, 1.63it/s]--------------------------------------------- Result 69 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (90%)\u001b[0m\n", + "\n", + "despite its faults , gangs \u001b[92mexcels\u001b[0m in spectacle and pacing .\n", + "\n", + "despite its faults , gangs \u001b[91moverwhelms\u001b[0m in spectacle and pacing .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 54 / 1 / 14 / 69: 70% 70/100 [00:42<00:18, 1.66it/s]--------------------------------------------- Result 70 ---------------------------------------------\n", + "\u001b[92mPositive (80%)\u001b[0m --> \u001b[91mNegative (51%)\u001b[0m\n", + "\n", + "\u001b[92mentertaining\u001b[0m despite its one-joke premise with the thesis that women from venus and men from mars can indeed get together .\n", + "\n", + "\u001b[91mamusing\u001b[0m despite its one-joke premise with the thesis that women from venus and men from mars can indeed get together .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 55 / 1 / 14 / 70: 70% 70/100 [00:42<00:18, 1.66it/s]--------------------------------------------- Result 71 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (91%)\u001b[0m\n", + "\n", + "a tightly directed , \u001b[92mhighly\u001b[0m professional film that's old-fashioned in all the best possible ways .\n", + "\n", + "a tightly directed , \u001b[91mexcessively\u001b[0m professional film that's old-fashioned in all the best possible ways .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 56 / 1 / 14 / 71: 72% 72/100 [00:43<00:16, 1.66it/s]--------------------------------------------- Result 72 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (60%)\u001b[0m\n", + "\n", + "it's \u001b[92mdark\u001b[0m but has \u001b[92mwonderfully\u001b[0m \u001b[92mfunny\u001b[0m \u001b[92mmoments\u001b[0m ; you \u001b[92mcare\u001b[0m about the characters ; and the \u001b[92maction\u001b[0m and special effects are first-rate .\n", + "\n", + "it's \u001b[91mghoulish\u001b[0m but has \u001b[91munspeakably\u001b[0m \u001b[91mjoke\u001b[0m \u001b[91mmins\u001b[0m ; you \u001b[91mzorg\u001b[0m about the characters ; and the \u001b[91moperating\u001b[0m and special effects are first-rate .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 57 / 1 / 14 / 72: 72% 72/100 [00:43<00:16, 1.66it/s]--------------------------------------------- Result 73 ---------------------------------------------\n", + "\u001b[92mPositive (89%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", + "\n", + "in visual fertility \u001b[92mtreasure\u001b[0m planet rivals the \u001b[92mtop\u001b[0m japanese animations of recent vintage .\n", + "\n", + "in visual fertility \u001b[91mcoffer\u001b[0m planet rivals the \u001b[91msupremo\u001b[0m japanese animations of recent vintage .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 58 / 1 / 14 / 73: 74% 74/100 [00:44<00:15, 1.68it/s]--------------------------------------------- Result 74 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (96%)\u001b[0m\n", + "\n", + "enormously \u001b[92menjoyable\u001b[0m , high-adrenaline \u001b[92mdocumentary\u001b[0m .\n", + "\n", + "enormously \u001b[91mdroll\u001b[0m , high-adrenaline \u001b[91mpaperwork\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 59 / 1 / 14 / 74: 74% 74/100 [00:44<00:15, 1.68it/s]--------------------------------------------- Result 75 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (50%)\u001b[0m\n", + "\n", + "buy is an accomplished actress , and this is a big , \u001b[92mjuicy\u001b[0m role .\n", + "\n", + "buy is an accomplished actress , and this is a big , \u001b[91mcrusty\u001b[0m role .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 60 / 1 / 14 / 75: 76% 76/100 [00:44<00:14, 1.69it/s]--------------------------------------------- Result 76 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", + "\n", + "it \u001b[92mworks\u001b[0m its \u001b[92mmagic\u001b[0m with such \u001b[92mexuberance\u001b[0m and passion that the film's length becomes a part of its \u001b[92mfun\u001b[0m .\n", + "\n", + "it \u001b[91mfunctioned\u001b[0m its \u001b[91mpotions\u001b[0m with such \u001b[91melation\u001b[0m and passion that the film's length becomes a part of its \u001b[91mbanter\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 61 / 1 / 14 / 76: 76% 76/100 [00:44<00:14, 1.69it/s]--------------------------------------------- Result 77 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", + "\n", + "\u001b[92mbeautifully\u001b[0m crafted and \u001b[92mbrutally\u001b[0m \u001b[92mhonest\u001b[0m , promises offers an \u001b[92munexpected\u001b[0m \u001b[92mwindow\u001b[0m into the complexities of the middle east \u001b[92mstruggle\u001b[0m and into the \u001b[92mhumanity\u001b[0m of its people .\n", + "\n", + "\u001b[91mimpossibly\u001b[0m crafted and \u001b[91mhastily\u001b[0m \u001b[91mveritable\u001b[0m , promises offers an \u001b[91munforeseen\u001b[0m \u001b[91mfibreglass\u001b[0m into the complexities of the middle east \u001b[91mtussle\u001b[0m and into the \u001b[91mhumans\u001b[0m of its people .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 62 / 1 / 14 / 77: 78% 78/100 [00:46<00:13, 1.68it/s]--------------------------------------------- Result 78 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (67%)\u001b[0m\n", + "\n", + "an old-fashioned but emotionally \u001b[92mstirring\u001b[0m adventure tale of the kind they rarely make anymore .\n", + "\n", + "an old-fashioned but emotionally \u001b[91mwavering\u001b[0m adventure tale of the kind they rarely make anymore .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 63 / 1 / 14 / 78: 78% 78/100 [00:46<00:13, 1.68it/s]--------------------------------------------- Result 79 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (93%)\u001b[0m\n", + "\n", + "charlotte sometimes is a \u001b[92mgem\u001b[0m . it's always \u001b[92menthralling\u001b[0m .\n", + "\n", + "charlotte sometimes is a \u001b[91mbling\u001b[0m . it's always \u001b[91mhallucinatory\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 64 / 1 / 14 / 79: 80% 80/100 [00:47<00:11, 1.70it/s]--------------------------------------------- Result 80 ---------------------------------------------\n", + "\u001b[92mPositive (92%)\u001b[0m --> \u001b[91mNegative (50%)\u001b[0m\n", + "\n", + "in my opinion , analyze that is not as funny or entertaining as \u001b[92manalyze\u001b[0m this , but it is a \u001b[92mrespectable\u001b[0m sequel .\n", + "\n", + "in my opinion , analyze that is not as funny or entertaining as \u001b[91mdiscusses\u001b[0m this , but it is a \u001b[91mreputable\u001b[0m sequel .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 65 / 1 / 14 / 80: 80% 80/100 [00:47<00:11, 1.70it/s]--------------------------------------------- Result 81 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (94%)\u001b[0m\n", + "\n", + "a \u001b[92mremarkable\u001b[0m film by bernard rose .\n", + "\n", + "a \u001b[91mwhopping\u001b[0m film by bernard rose .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 66 / 1 / 14 / 81: 82% 82/100 [00:47<00:10, 1.72it/s]--------------------------------------------- Result 82 ---------------------------------------------\n", + "\u001b[92mPositive (80%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", + "\n", + "zhuangzhuang creates delicate balance of style , text , and subtext that's so simple and \u001b[92mprecise\u001b[0m that anything discordant would topple the balance , but against all odds , nothing does .\n", + "\n", + "zhuangzhuang creates delicate balance of style , text , and subtext that's so simple and \u001b[91mspecify\u001b[0m that anything discordant would topple the balance , but against all odds , nothing does .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 67 / 1 / 14 / 82: 82% 82/100 [00:47<00:10, 1.72it/s]--------------------------------------------- Result 83 ---------------------------------------------\n", + "\u001b[92mPositive (84%)\u001b[0m --> \u001b[91mNegative (71%)\u001b[0m\n", + "\n", + "a much more \u001b[92msuccessful\u001b[0m translation than its most famous previous film adaptation , writer-director anthony friedman's similarly updated 1970 british production .\n", + "\n", + "a much more \u001b[91mpropitious\u001b[0m translation than its most famous previous film adaptation , writer-director anthony friedman's similarly updated 1970 british production .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 68 / 1 / 14 / 83: 84% 84/100 [00:47<00:09, 1.75it/s]--------------------------------------------- Result 84 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (56%)\u001b[0m\n", + "\n", + "an \u001b[92moriginal\u001b[0m and highly cerebral examination of the psychopathic mind\n", + "\n", + "an \u001b[91mrudimentary\u001b[0m and highly cerebral examination of the psychopathic mind\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 69 / 1 / 14 / 84: 84% 84/100 [00:47<00:09, 1.75it/s]--------------------------------------------- Result 85 ---------------------------------------------\n", + "\u001b[92mPositive (93%)\u001b[0m --> \u001b[91mNegative (83%)\u001b[0m\n", + "\n", + "michel piccoli's \u001b[92mmoving\u001b[0m performance is this films reason for being .\n", + "\n", + "michel piccoli's \u001b[91mresettled\u001b[0m performance is this films reason for being .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 70 / 1 / 14 / 85: 86% 86/100 [00:48<00:07, 1.76it/s]--------------------------------------------- Result 86 ---------------------------------------------\n", + "\u001b[92mPositive (99%)\u001b[0m --> \u001b[91mNegative (95%)\u001b[0m\n", + "\n", + "a \u001b[92mcaptivating\u001b[0m and \u001b[92mintimate\u001b[0m \u001b[92mstudy\u001b[0m about \u001b[92mdying\u001b[0m and loving . . .\n", + "\n", + "a \u001b[91mhallucinatory\u001b[0m and \u001b[91mcosy\u001b[0m \u001b[91mscrutinized\u001b[0m about \u001b[91mdecedent\u001b[0m and loving . . .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 71 / 1 / 14 / 86: 86% 86/100 [00:48<00:07, 1.76it/s]--------------------------------------------- Result 87 ---------------------------------------------\n", + "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (75%)\u001b[0m\n", + "\n", + "this is an \u001b[92melegantly\u001b[0m \u001b[92mbalanced\u001b[0m movie -- every member of the ensemble has something fascinating to do -- that doesn't reveal even a hint of artifice .\n", + "\n", + "this is an \u001b[91mprettily\u001b[0m \u001b[91mbalancing\u001b[0m movie -- every member of the ensemble has something fascinating to do -- that doesn't reveal even a hint of artifice .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 72 / 1 / 14 / 87: 88% 88/100 [00:49<00:06, 1.78it/s]--------------------------------------------- Result 88 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (61%)\u001b[0m\n", + "\n", + "[grant] goes beyond his usual fluttering and stammering and \u001b[92mcaptures\u001b[0m the \u001b[92msoul\u001b[0m of a man in pain who gradually comes to recognize it and deal with it .\n", + "\n", + "[grant] goes beyond his usual fluttering and stammering and \u001b[91mincarcerate\u001b[0m the \u001b[91mwits\u001b[0m of a man in pain who gradually comes to recognize it and deal with it .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 73 / 1 / 14 / 88: 88% 88/100 [00:49<00:06, 1.78it/s]--------------------------------------------- Result 89 ---------------------------------------------\n", + "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (58%)\u001b[0m\n", + "\n", + "a high-spirited buddy \u001b[92mmovie\u001b[0m about the \u001b[92mreunion\u001b[0m of \u001b[92mberlin\u001b[0m \u001b[92manarchists\u001b[0m who \u001b[92mface\u001b[0m \u001b[92marrest\u001b[0m 15 \u001b[92myears\u001b[0m after their \u001b[92mcrime\u001b[0m .\n", + "\n", + "a high-spirited buddy \u001b[91mvideo\u001b[0m about the \u001b[91mpooled\u001b[0m of \u001b[91mgermania\u001b[0m \u001b[91manarchist\u001b[0m who \u001b[91mfacial\u001b[0m \u001b[91mintercepted\u001b[0m 15 \u001b[91molds\u001b[0m after their \u001b[91mpenal\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 74 / 1 / 14 / 89: 90% 90/100 [00:50<00:05, 1.77it/s]--------------------------------------------- Result 90 ---------------------------------------------\n", + "\u001b[91mNegative (84%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "about the best thing you could say about narc is that it's a rock-solid little genre picture . whether you like it or not is basically a matter of taste .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 74 / 1 / 15 / 90: 90% 90/100 [00:50<00:05, 1.77it/s]--------------------------------------------- Result 91 ---------------------------------------------\n", + "\u001b[92mPositive (97%)\u001b[0m --> \u001b[91mNegative (80%)\u001b[0m\n", + "\n", + "an involving , \u001b[92minspirational\u001b[0m \u001b[92mdrama\u001b[0m that sometimes falls prey to its sob-story trappings .\n", + "\n", + "an involving , \u001b[91mincentive\u001b[0m \u001b[91mcataclysmic\u001b[0m that sometimes falls prey to its sob-story trappings .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 75 / 1 / 15 / 91: 92% 92/100 [00:51<00:04, 1.79it/s]--------------------------------------------- Result 92 ---------------------------------------------\n", + "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (75%)\u001b[0m\n", + "\n", + "some of the most \u001b[92minventive\u001b[0m silliness you are likely to witness in a movie theatre for some time .\n", + "\n", + "some of the most \u001b[91mcontrivance\u001b[0m silliness you are likely to witness in a movie theatre for some time .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 76 / 1 / 15 / 92: 92% 92/100 [00:51<00:04, 1.79it/s]--------------------------------------------- Result 93 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (57%)\u001b[0m\n", + "\n", + "canadian \u001b[92mfilmmaker\u001b[0m gary burns' \u001b[92minventive\u001b[0m and mordantly \u001b[92mhumorous\u001b[0m \u001b[92mtake\u001b[0m on the soullessness of work in the \u001b[92mcity\u001b[0m .\n", + "\n", + "canadian \u001b[91mscriptwriter\u001b[0m gary burns' \u001b[91minventor\u001b[0m and mordantly \u001b[91mprank\u001b[0m \u001b[91mtakes\u001b[0m on the soullessness of work in the \u001b[91mshing\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 77 / 1 / 15 / 93: 94% 94/100 [00:52<00:03, 1.78it/s]--------------------------------------------- Result 94 ---------------------------------------------\n", + "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (65%)\u001b[0m\n", + "\n", + "a rollicking \u001b[92mride\u001b[0m , with jaw-dropping action sequences , striking villains , a \u001b[92mgorgeous\u001b[0m color palette , astounding technology , \u001b[92mstirring\u001b[0m music and a boffo last hour that leads up to a strangely sinister happy ending .\n", + "\n", + "a rollicking \u001b[91mwrinkle\u001b[0m , with jaw-dropping action sequences , striking villains , a \u001b[91mleggy\u001b[0m color palette , astounding technology , \u001b[91magitation\u001b[0m music and a boffo last hour that leads up to a strangely sinister happy ending .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 78 / 1 / 15 / 94: 94% 94/100 [00:52<00:03, 1.78it/s]--------------------------------------------- Result 95 ---------------------------------------------\n", + "\u001b[92mPositive (99%)\u001b[0m --> \u001b[91mNegative (51%)\u001b[0m\n", + "\n", + "everyone's insecure in lovely and \u001b[92mamazing\u001b[0m , a \u001b[92mpoignant\u001b[0m and wryly amusing film about mothers , daughters and their relationships .\n", + "\n", + "everyone's insecure in lovely and \u001b[91mwhopping\u001b[0m , a \u001b[91mdisquieting\u001b[0m and wryly amusing film about mothers , daughters and their relationships .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 79 / 1 / 15 / 95: 96% 96/100 [00:53<00:02, 1.80it/s]--------------------------------------------- Result 96 ---------------------------------------------\n", + "\u001b[92mPositive (52%)\u001b[0m --> \u001b[91mNegative (75%)\u001b[0m\n", + "\n", + "the closest thing to the \u001b[92mexperience\u001b[0m of space travel\n", + "\n", + "the closest thing to the \u001b[91mpilot\u001b[0m of space travel\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 80 / 1 / 15 / 96: 96% 96/100 [00:53<00:02, 1.80it/s]--------------------------------------------- Result 97 ---------------------------------------------\n", + "\u001b[92mPositive (96%)\u001b[0m --> \u001b[91mNegative (96%)\u001b[0m\n", + "\n", + "full of \u001b[92msurprises\u001b[0m .\n", + "\n", + "full of \u001b[91mstumped\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 81 / 1 / 15 / 97: 98% 98/100 [00:54<00:01, 1.81it/s]--------------------------------------------- Result 98 ---------------------------------------------\n", + "\u001b[92mPositive (90%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n", + "\n", + "connoisseurs of \u001b[92mchinese\u001b[0m film will be pleased to discover that tian's meticulous \u001b[92mtalent\u001b[0m \u001b[92mhas\u001b[0m not \u001b[92mwithered\u001b[0m during his enforced \u001b[92mhiatus\u001b[0m .\n", + "\n", + "connoisseurs of \u001b[91mcantonese\u001b[0m film will be pleased to discover that tian's meticulous \u001b[91mstaffing\u001b[0m \u001b[91mis\u001b[0m not \u001b[91mbloomed\u001b[0m during his enforced \u001b[91mharford\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 82 / 1 / 15 / 98: 98% 98/100 [00:54<00:01, 1.81it/s]--------------------------------------------- Result 99 ---------------------------------------------\n", + "\u001b[92mPositive (95%)\u001b[0m --> \u001b[91mNegative (69%)\u001b[0m\n", + "\n", + "if you can push on through the slow spots , you'll be \u001b[92mrewarded\u001b[0m with some \u001b[92mfine\u001b[0m \u001b[92macting\u001b[0m .\n", + "\n", + "if you can push on through the slow spots , you'll be \u001b[91mrecompense\u001b[0m with some \u001b[91mwondrous\u001b[0m \u001b[91mbehaving\u001b[0m .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 83 / 1 / 15 / 99: 100% 100/100 [00:54<00:00, 1.84it/s]--------------------------------------------- Result 100 ---------------------------------------------\n", + "\u001b[91mNegative (50%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n", + "\n", + "an unusually dry-eyed , even analytical approach to material that is generally played for maximum moisture .\n", + "\n", + "\n", + "[Succeeded / Failed / Skipped / Total] 83 / 1 / 16 / 100: 100% 100/100 [00:54<00:00, 1.84it/s]\n", + "\n", + "+-------------------------------+--------+\n", + "| Attack Results | |\n", + "+-------------------------------+--------+\n", + "| Number of successful attacks: | 83 |\n", + "| Number of failed attacks: | 1 |\n", + "| Number of skipped attacks: | 16 |\n", + "| Original accuracy: | 84.0% |\n", + "| Accuracy under attack: | 1.0% |\n", + "| Attack success rate: | 98.81% |\n", + "| Average perturbed word %: | 13.68% |\n", + "| Average num. words per input: | 18.45 |\n", + "| Avg num queries: | 80.13 |\n", + "+-------------------------------+--------+\n" + ] } - ] -} \ No newline at end of file + ], + "source": [ + "!textattack attack --recipe textfooler --num-examples 100 --model ./outputs/2021-10-13-17-37-27-247436/best_model/ --dataset-from-huggingface rotten_tomatoes --dataset-split test" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iyrJM3CaseoL" + }, + "source": [ + "Looks like our model was 84% successful (makes sense - same evaluation set as `textattack eval`!), meaning that TextAttack attacked the model with 84 examples (since the attack won't run if an example is originally mispredicted). The attack success rate was 98.8%, meaning that TextFooler failed to find an adversarial example only 1.2% (1 out of 84) of the time.\n", + "\n", + "\n", + "## Conclusion\n", + "\n", + "That's all, folks! We've learned how to train, evaluate, and attack a model with TextAttack, using only three commands! 😀\n", + "\n", + "\n", + "\n", + "## Bonus\n", + "\n", + "There are many powerful functions in TextAttack, we can use through command lines. Here is a list of examples as bonus for your learning. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!textattack attack --recipe deepwordbug --model lstm-mr --num-examples 2 --log-summary-to-json attack_summary.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!textattack attack --model cnn-yelp --num-examples 3 --search-method greedy-word-wir --transformation word-swap-wordnet --constraints cola^max_diff=0.1 bert-score^min_bert_score=0.7 --enable-advance-metrics \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!textattack attack --model lstm-mr --recipe deepwordbug --num-examples 2 --attack-n --enable-advance-metrics \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!textattack attack --model lstm-mr --recipe hotflip --num-examples 4 --num-examples-offset 3 --enable-advance-metrics " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!textattack attack --model-from-huggingface distilbert-base-uncased-finetuned-sst-2-english --dataset-from-huggingface glue^sst2^train --recipe deepwordbug --num-examples 3 --enable-advance-metrics\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! textattack attack --model cnn-imdb --attack-from-file tests/sample_inputs/attack_from_file.py^Attack --num-examples 2 --num-examples-offset 18 --attack-n " + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "0_End_to_End.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.11" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/docs/2notebook/1_Introduction_and_Transformations.ipynb b/docs/2notebook/1_Introduction_and_Transformations.ipynb index 6a4db7ec3..b895702fd 100644 --- a/docs/2notebook/1_Introduction_and_Transformations.ipynb +++ b/docs/2notebook/1_Introduction_and_Transformations.ipynb @@ -927,7 +927,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.7.11" } }, "nbformat": 4, diff --git a/docs/2notebook/2_Constraints.ipynb b/docs/2notebook/2_Constraints.ipynb index b219ca2c3..261c8f124 100644 --- a/docs/2notebook/2_Constraints.ipynb +++ b/docs/2notebook/2_Constraints.ipynb @@ -76,15 +76,6 @@ "Let's import NLTK and download the required modules:" ] }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# cd .." - ] - }, { "cell_type": "code", "execution_count": 11, @@ -147,7 +138,7 @@ } ], "source": [ - "!pip3 install .\n", + "! pip3 install textattack[tensorflow]\n", "\n", "import nltk\n", "\n", @@ -826,7 +817,7 @@ ")\n", "attacker = Attacker(attack, dataset, attack_args)\n", "\n", - "attacker.attack_dataset()" + "attack_results = attacker.attack_dataset()" ] }, { @@ -884,7 +875,7 @@ "provenance": [] }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -898,7 +889,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.7.11" }, "widgets": { "application/vnd.jupyter.widget-state+json": { diff --git a/docs/2notebook/3_Augmentations.ipynb b/docs/2notebook/3_Augmentations.ipynb index f136fe609..4a3b98054 100644 --- a/docs/2notebook/3_Augmentations.ipynb +++ b/docs/2notebook/3_Augmentations.ipynb @@ -1,29 +1,4 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "Augmentation with TextAttack.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" - } - }, "cells": [ { "cell_type": "markdown", @@ -84,9 +59,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "5AXyxiLD4X93" }, + "outputs": [], "source": [ "# import transformations, contraints, and the Augmenter\n", "from textattack.transformations import WordSwapRandomCharacterDeletion\n", @@ -97,12 +74,11 @@ "from textattack.constraints.pre_transformation import StopwordModification\n", "\n", "from textattack.augmentation import Augmenter" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -110,25 +86,6 @@ "id": "wFeXF_OL-vyw", "outputId": "c041e77e-accd-4a58-88be-9b140dd0cd56" }, - "source": [ - "# Set up transformation using CompositeTransformation()\n", - "transformation = CompositeTransformation(\n", - " [WordSwapRandomCharacterDeletion(), WordSwapQWERTY()]\n", - ")\n", - "# Set up constraints\n", - "constraints = [RepeatModification(), StopwordModification()]\n", - "# Create augmenter with specified parameters\n", - "augmenter = Augmenter(\n", - " transformation=transformation,\n", - " constraints=constraints,\n", - " pct_words_to_swap=0.5,\n", - " transformations_per_example=10,\n", - ")\n", - "s = \"What I cannot create, I do not understand.\"\n", - "# Augment!\n", - "augmenter.augment(s)" - ], - "execution_count": null, "outputs": [ { "data": { @@ -149,6 +106,24 @@ "metadata": {}, "output_type": "execute_result" } + ], + "source": [ + "# Set up transformation using CompositeTransformation()\n", + "transformation = CompositeTransformation(\n", + " [WordSwapRandomCharacterDeletion(), WordSwapQWERTY()]\n", + ")\n", + "# Set up constraints\n", + "constraints = [RepeatModification(), StopwordModification()]\n", + "# Create augmenter with specified parameters\n", + "augmenter = Augmenter(\n", + " transformation=transformation,\n", + " constraints=constraints,\n", + " pct_words_to_swap=0.5,\n", + " transformations_per_example=10,\n", + ")\n", + "s = \"What I cannot create, I do not understand.\"\n", + "# Augment!\n", + "augmenter.augment(s)" ] }, { @@ -173,6 +148,7 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -180,17 +156,6 @@ "id": "WkYiVH6lQedu", "outputId": "cd5ffc65-ca80-45cd-b3bb-d023bcad09a4" }, - "source": [ - "# import the CheckListAugmenter\n", - "from textattack.augmentation import CheckListAugmenter\n", - "\n", - "# Alter default values if desired\n", - "augmenter = CheckListAugmenter(pct_words_to_swap=0.2, transformations_per_example=5)\n", - "s = \"I'd love to go to Japan but the tickets are 500 dollars\"\n", - "# Augment\n", - "augmenter.augment(s)" - ], - "execution_count": null, "outputs": [ { "name": "stdout", @@ -218,6 +183,16 @@ "metadata": {}, "output_type": "execute_result" } + ], + "source": [ + "# import the CheckListAugmenter\n", + "from textattack.augmentation import CheckListAugmenter\n", + "\n", + "# Alter default values if desired\n", + "augmenter = CheckListAugmenter(pct_words_to_swap=0.2, transformations_per_example=5)\n", + "s = \"I'd love to go to Japan but the tickets are 500 dollars\"\n", + "# Augment\n", + "augmenter.augment(s)" ] }, { @@ -248,6 +223,7 @@ }, { "cell_type": "code", + "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -255,35 +231,17 @@ "id": "l2b-4scuXvkA", "outputId": "5a372fd2-226a-4970-a2c9-c09bf2af56c2" }, - "source": [ - "from textattack.augmentation import WordNetAugmenter\n", - "\n", - "augmenter = WordNetAugmenter(\n", - " pct_words_to_swap=0.4,\n", - " transformations_per_example=5,\n", - " high_yield=True,\n", - " enable_advanced_metrics=True,\n", - ")\n", - "s = \"I'd love to go to Japan but the tickets are 500 dollars\"\n", - "results = augmenter.augment(s)\n", - "print(f\"Average Original Perplexity Score: {results[1]['avg_original_perplexity']}\\n\")\n", - "print(f\"Average Augment Perplexity Score: {results[1]['avg_attack_perplexity']}\\n\")\n", - "print(f\"Average Augment USE Score: {results[2]['avg_attack_use_score']}\\n\")\n", - "print(f\"Augmentations:\")\n", - "results[0]" - ], - "execution_count": 9, "outputs": [ { - "output_type": "stream", "name": "stderr", + "output_type": "stream", "text": [ "Token indices sequence length is longer than the specified maximum sequence length for this model (1091 > 1024). Running this sequence through the model will result in indexing errors\n" ] }, { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Average Original Perplexity Score: 1.09\n", "\n", @@ -295,7 +253,6 @@ ] }, { - "output_type": "execute_result", "data": { "text/plain": [ "[\"I'd bang to operate to Japan but the ticket are 500 buck\",\n", @@ -373,9 +330,27 @@ " \"I'd screw to plump to Nihon but the tickets are 500 clam\"]" ] }, + "execution_count": 9, "metadata": {}, - "execution_count": 9 + "output_type": "execute_result" } + ], + "source": [ + "from textattack.augmentation import WordNetAugmenter\n", + "\n", + "augmenter = WordNetAugmenter(\n", + " pct_words_to_swap=0.4,\n", + " transformations_per_example=5,\n", + " high_yield=True,\n", + " enable_advanced_metrics=True,\n", + ")\n", + "s = \"I'd love to go to Japan but the tickets are 500 dollars\"\n", + "results = augmenter.augment(s)\n", + "print(f\"Average Original Perplexity Score: {results[1]['avg_original_perplexity']}\\n\")\n", + "print(f\"Average Augment Perplexity Score: {results[1]['avg_attack_perplexity']}\\n\")\n", + "print(f\"Average Augment USE Score: {results[2]['avg_attack_use_score']}\\n\")\n", + "print(f\"Augmentations:\")\n", + "results[0]" ] }, { @@ -388,5 +363,30 @@ "We have now went through the basics in running `Augmenter` by either creating a new augmenter from scratch or using a pre-built augmenter. This could be done in as few as 4 lines of code so please give it a try if you haven't already! 🐙" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "name": "Augmentation with TextAttack.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.11" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/docs/2notebook/4_Custom_Datasets_Word_Embedding.ipynb b/docs/2notebook/4_Custom_Datasets_Word_Embedding.ipynb index 506b705ac..b9639346f 100644 --- a/docs/2notebook/4_Custom_Datasets_Word_Embedding.ipynb +++ b/docs/2notebook/4_Custom_Datasets_Word_Embedding.ipynb @@ -537,10 +537,31 @@ "\n", "attack = Attack(goal_function, constraints, transformation, search_method)\n", "\n", + "# here is a legacy code piece showing how the attack runs in details \n", "for example, label in custom_dataset:\n", " result = attack.attack(example, label)\n", " print(result.__str__(color_method=\"ansi\"))" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# here is currently recommendated API-centric way to use customized attack\n", + "\n", + "from textattack.loggers import CSVLogger # tracks a dataframe for us.\n", + "from textattack.attack_results import SuccessfulAttackResult\n", + "from textattack import Attacker, AttackArgs\n", + "\n", + "attack_args = AttackArgs(\n", + " num_successful_examples=5, log_to_csv=\"results.csv\", csv_coloring_style=\"html\"\n", + ")\n", + "attacker = Attacker(attack, dataset, attack_args)\n", + "\n", + "attack_results = attacker.attack_dataset()" + ] } ], "metadata": { @@ -549,7 +570,7 @@ "provenance": [] }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -563,7 +584,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.7.11" } }, "nbformat": 4, diff --git a/docs/2notebook/Example_6_Chinese_Attack.ipynb b/docs/2notebook/Example_6_Chinese_Attack.ipynb index 66e93918f..8b2869d62 100644 --- a/docs/2notebook/Example_6_Chinese_Attack.ipynb +++ b/docs/2notebook/Example_6_Chinese_Attack.ipynb @@ -1,1595 +1,795 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "m83IiqVREJ96" + }, + "source": [ + "# Adapting TextAttack to Chinese Language" + ] }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" + { + "cell_type": "markdown", + "metadata": { + "id": "6UZ0d84hEJ98" + }, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/QData/TextAttack/blob/master/docs/2notebook/Example_6_Chinese%20Attack.ipynb)\n", + "\n", + "\n", + "[![View Source on GitHub](https://img.shields.io/badge/github-view%20source-black.svg)](https://github.com/QData/TextAttack/blob/master/docs/2notebook/Example_6_Chinese%20Attack.ipynb)" + ] }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 + { + "cell_type": "markdown", + "metadata": { + "id": "tjqc2c5_7YaX" }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" + "source": [ + " Please remember to run the following in your notebook enviroment before running the tutorial codes:\n", + "\n", + "```\n", + "pip3 install textattack[tensorflow]\n", + "```\n", + "\n", + "\n", + "\n" + ] }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "4b423038915e40158f9da4c07d09aad3": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_3711cf0a18994cee8fc840d9a93cf5d3", - "IPY_MODEL_7f77bd7b8e5f45ae94cfc45f915c0c72", - "IPY_MODEL_fe0ca6138bc54b628c03e590c6e96aed" - ], - "layout": "IPY_MODEL_8b39363f69eb46009c5357263a65248c" - } - }, - "3711cf0a18994cee8fc840d9a93cf5d3": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6b976fd913584da69456c1b6d53483cb", - "placeholder": "​", - "style": "IPY_MODEL_ea568ab2407f474da3b1f1b2540fa3a8", - "value": "Downloading: 100%" - } - }, - "7f77bd7b8e5f45ae94cfc45f915c0c72": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ff6b34a7e75b443593f3dca5d050cd52", - "max": 615, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_4f31972fd2fd44bbac063bb4b5075e98", - "value": 615 - } - }, - "fe0ca6138bc54b628c03e590c6e96aed": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7de1551891ec447ab6d80ea1de145f16", - "placeholder": "​", - "style": "IPY_MODEL_e5e2c0507c834887b80f5717c1e6d5f3", - "value": " 615/615 [00:00<00:00, 33.8kB/s]" - } - }, - "8b39363f69eb46009c5357263a65248c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6b976fd913584da69456c1b6d53483cb": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ea568ab2407f474da3b1f1b2540fa3a8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ff6b34a7e75b443593f3dca5d050cd52": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4f31972fd2fd44bbac063bb4b5075e98": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "7de1551891ec447ab6d80ea1de145f16": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e5e2c0507c834887b80f5717c1e6d5f3": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "588b1321a9274de6a8a9e86622d90be4": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_2436b07259a34ee18fe9c1007f7b615b", - "IPY_MODEL_98aac5a0baee4930bd461f2c5fd73f4a", - "IPY_MODEL_34607a8556794a5a86c18abe5bd7e5a5" - ], - "layout": "IPY_MODEL_f78f6701ce4f4b3b9ff0af925620f261" - } - }, - "2436b07259a34ee18fe9c1007f7b615b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a1e3fb5cceed4e95957a17192a641b69", - "placeholder": "​", - "style": "IPY_MODEL_83e9b14c4d354fdc80db4f8a881f19f3", - "value": "Downloading: 100%" - } - }, - "98aac5a0baee4930bd461f2c5fd73f4a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5f5457f292284dd8b914f45e26b2f749", - "max": 1115590446, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_2bb72191846f49528663680a315d8b01", - "value": 1115590446 - } - }, - "34607a8556794a5a86c18abe5bd7e5a5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_83eff532314e4edcbfe648b321e9a310", - "placeholder": "​", - "style": "IPY_MODEL_3d30e700d32443fdb37b5ab934d2d70a", - "value": " 1.04G/1.04G [00:25<00:00, 45.4MB/s]" - } - }, - "f78f6701ce4f4b3b9ff0af925620f261": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a1e3fb5cceed4e95957a17192a641b69": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "cell_type": "markdown", + "metadata": { + "id": "qZ5xnoevEJ99" + }, + "source": [ + "With a few additional modifications to the standard TextAttack commands, lanaguage models in Chinese can be attacked just as English models. Four transformations are available for either Chinese attack or augmentation:\n", + "\n", + "1. **ChineseHomophoneCharacterSwap**: transforms an input by replacing its words with substitions that share similar/identical pronounciation.\n", + "2. **ChineseMorphonymCharacterSwap**: transforms an input by replacing its words with substitions that share similar glyph structures.\n", + "3. **ChineseWordSwapHowNet**: transforms an input by replacing its words with synonyms provided by [OpenHownet](http://nlp.csai.tsinghua.edu.cn/).\n", + "4. **ChineseWordSwapMaskedLM**: transforms an input with potential replacements using a masked language model." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2EP1DJylSfkD" + }, + "source": [ + "We begin with imports:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5AXyxiLD4X93" + }, + "outputs": [], + "source": [ + "# Import required packages\n", + "import transformers\n", + "import string\n", + "import os\n", + "import pandas as pd\n", + "import datasets\n", + "\n", + "# Import classes required to build an Attacker\n", + "from textattack.models.wrappers import HuggingFaceModelWrapper\n", + "from textattack.search_methods import GreedyWordSwapWIR\n", + "from textattack.constraints.pre_transformation import (\n", + " RepeatModification,\n", + " StopwordModification,\n", + ")\n", + "from textattack.goal_functions import UntargetedClassification\n", + "\n", + "from textattack import Attack, Attacker, AttackArgs\n", + "from textattack.loggers import CSVLogger\n", + "from textattack.datasets import Dataset, HuggingFaceDataset\n", + "\n", + "# Import optional MUSE for higher quality examples\n", + "from textattack.constraints.semantics.sentence_encoders import (\n", + " MultilingualUniversalSentenceEncoder,\n", + ")\n", + "\n", + "muse = MultilingualUniversalSentenceEncoder(\n", + " threshold=0.9,\n", + " metric=\"cosine\",\n", + " compare_against_original=True,\n", + " window_size=15,\n", + " skip_text_shorter_than_window=True,\n", + ")\n", + "\n", + "# Import the transformations\n", + "\n", + "from textattack.transformations import CompositeTransformation\n", + "from textattack.transformations import ChineseWordSwapMaskedLM\n", + "from textattack.transformations import ChineseMorphonymCharacterSwap\n", + "from textattack.transformations import ChineseWordSwapHowNet\n", + "from textattack.transformations import ChineseHomophoneCharacterSwap" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1mSvCqhHSi0h" + }, + "source": [ + "Models and datasets would also need to be set up:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CfnC9qUFPq9h" + }, + "outputs": [], + "source": [ + "# In this example, we will attack a pre-trained entailment model from HugginFace (https://huggingface.co/uer/roberta-base-finetuned-chinanews-chinese)\n", + "tokenizer = transformers.AutoTokenizer.from_pretrained(\n", + " \"uer/roberta-base-finetuned-chinanews-chinese\"\n", + ")\n", + "model = transformers.AutoModelForSequenceClassification.from_pretrained(\n", + " \"uer/roberta-base-finetuned-chinanews-chinese\"\n", + ")\n", + "model_wrapper = HuggingFaceModelWrapper(model, tokenizer)\n", + "\n", + "# Set goal function\n", + "goal_function = UntargetedClassification(model_wrapper, query_budget=10000)\n", + "\n", + "# Set dataset from which we will generate adversraial examples\n", + "path = os.path.abspath(\"\")\n", + "path_list = path.split(os.sep)\n", + "temppath = os.path.normpath(\"examples/dataset/zh_sentiment/entailment_dataset.tsv\")\n", + "dataset = datasets.load_dataset(\"csv\", data_files=temppath, delimiter=\"\\t\")[\"train\"]\n", + "dataset = HuggingFaceDataset(\n", + " dataset,\n", + " dataset_columns=([\"text\"], \"label\"),\n", + " label_names=[\n", + " \"Mainland China politics\",\n", + " \"Hong Kong - Macau politics\",\n", + " \"International news\",\n", + " \"Financial news\",\n", + " \"Culture\",\n", + " \"Entertainment\",\n", + " \"Sports\",\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XfJVzCdRSr3d" + }, + "source": [ + "If this is your first time running Hownet, run this code block" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Hgal-PHeQwys" + }, + "outputs": [], + "source": [ + "import OpenHowNet\n", + "\n", + "OpenHowNet.download()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SrtoxdrMSZ0X" + }, + "source": [ + "\n", + "\n", + "Now we are ready to attack! With goal function, transformation, constraints, search method, and goal function, we create the Attacker as any other TextAttack attacks\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "4b423038915e40158f9da4c07d09aad3", + "3711cf0a18994cee8fc840d9a93cf5d3", + "7f77bd7b8e5f45ae94cfc45f915c0c72", + "fe0ca6138bc54b628c03e590c6e96aed", + "8b39363f69eb46009c5357263a65248c", + "6b976fd913584da69456c1b6d53483cb", + "ea568ab2407f474da3b1f1b2540fa3a8", + "ff6b34a7e75b443593f3dca5d050cd52", + "4f31972fd2fd44bbac063bb4b5075e98", + "7de1551891ec447ab6d80ea1de145f16", + "e5e2c0507c834887b80f5717c1e6d5f3", + "588b1321a9274de6a8a9e86622d90be4", + "2436b07259a34ee18fe9c1007f7b615b", + "98aac5a0baee4930bd461f2c5fd73f4a", + "34607a8556794a5a86c18abe5bd7e5a5", + "f78f6701ce4f4b3b9ff0af925620f261", + "a1e3fb5cceed4e95957a17192a641b69", + "83e9b14c4d354fdc80db4f8a881f19f3", + "5f5457f292284dd8b914f45e26b2f749", + "2bb72191846f49528663680a315d8b01", + "83eff532314e4edcbfe648b321e9a310", + "3d30e700d32443fdb37b5ab934d2d70a", + "a132f09845a54cbe865cbe8159bb693e", + "0af0e1eaea2f48c5b0fec6e550bd1baa", + "dd6b0a5d9db245338a8fdb2ef5b29bf9", + "58fc309041b54e94ae265167fa20d8d7", + "89dfd3fdc41e417a870901bc79e47495", + "21472d1c4c8b494a8d3660b3320e9d4b", + "7511bb9ca5424674bb2350dff63c468a", + "f6dd2c2cb4e346fe9af7026b5d2162e9", + "a34ad57624fc422aa4832db3963298e6", + "5167daffe92e44d2acc2af2d9b9738df", + "acbfb34a353f41649675bd104069d14e", + "be070cb4a1624b0bb8f9b594c6b951a5", + "2edb7130713d4e10a07bbf808abb9771", + "5ae4c618f75d4ef9b65e5020fccb6d72", + "138d8260e67f4bc58106b9b42f7abd12", + "d7621b5c619a4ce38ebe63924374cf78", + "1b208b6df75f4a9e97faa4e3705a9442", + "a7871b8ec3ec40e7bbbe6a5f40b79f4a", + "aeb7ee752d834b4cbaa189419fd75dd4", + "b47dfff73e73410aa89f65e3c5b0c366", + "bdf3571e59ef4a688ab89d4badda27b1", + "d3bab427b92144d6b9ce96eac18ceb89" + ] }, - "83e9b14c4d354fdc80db4f8a881f19f3": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } + "id": "C_0Z8njnRblT", + "outputId": "3890d784-de7f-4b70-f984-cbc9e0c7f700" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4b423038915e40158f9da4c07d09aad3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading: 0%| | 0.00/615 [00:00 [[[FAILED]]]\n", + "\n", + "林书豪新秀赛上甘心\"跑龙套\" 自称仍是底薪球员\n", + "\n", + "\n" + ] }, - "dd6b0a5d9db245338a8fdb2ef5b29bf9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_f6dd2c2cb4e346fe9af7026b5d2162e9", - "max": 5069051, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_a34ad57624fc422aa4832db3963298e6", - "value": 5069051 - } + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 0 / 1 / 0 / 1: 10%|█ | 2/20 [06:55<1:02:18, 207.69s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 0 / 2 / 0 / 2: 10%|█ | 2/20 [06:55<1:02:18, 207.70s/it]\u001b[A" + ] }, - "58fc309041b54e94ae265167fa20d8d7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5167daffe92e44d2acc2af2d9b9738df", - "placeholder": "​", - "style": "IPY_MODEL_acbfb34a353f41649675bd104069d14e", - "value": " 4.83M/4.83M [00:00<00:00, 12.1MB/s]" - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 2 ---------------------------------------------\n", + "[[Culture (100%)]] --> [[[FAILED]]]\n", + "\n", + "成都现“真人图书馆”：无书“借人”给你读\n", + "\n", + "\n" + ] }, - "89dfd3fdc41e417a870901bc79e47495": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 0 / 2 / 0 / 2: 15%|█▌ | 3/20 [07:01<39:50, 140.61s/it] \u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 0 / 2 / 1 / 3: 15%|█▌ | 3/20 [07:01<39:50, 140.61s/it]\u001b[A" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 3 ---------------------------------------------\n", + "[[Mainland china politics (57%)]] --> [[[SKIPPED]]]\n", + "\n", + "中国经济走向更趋稳健务实\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 0 / 2 / 1 / 3: 20%|██ | 4/20 [11:33<46:12, 173.28s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 0 / 3 / 1 / 4: 20%|██ | 4/20 [11:33<46:12, 173.28s/it]\u001b[A" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 4 ---------------------------------------------\n", + "[[Sports (100%)]] --> [[[FAILED]]]\n", + "\n", + "国际田联世界挑战赛 罗伯斯迎来赛季第三冠\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 0 / 3 / 1 / 4: 25%|██▌ | 5/20 [14:52<44:36, 178.44s/it]\u001b[A" + ] }, - "21472d1c4c8b494a8d3660b3320e9d4b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 5 ---------------------------------------------\n" + ] }, - "7511bb9ca5424674bb2350dff63c468a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 1 / 5: 25%|██▌ | 5/20 [14:53<44:39, 178.62s/it]\u001b[A" + ] }, - "f6dd2c2cb4e346fe9af7026b5d2162e9": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[International news (66%)]] --> [[Entertainment (68%)]]\n", + "\n", + "德国一电视台合成“默克尔头巾照”惹争议\n", + "\n", + "德国一电视台合成“性感头巾照”惹争议\n", + "\n", + "\n" + ] }, - "a34ad57624fc422aa4832db3963298e6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 1 / 5: 30%|███ | 6/20 [14:57<34:55, 149.65s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 2 / 6: 30%|███ | 6/20 [14:57<34:55, 149.65s/it]\u001b[A" + ] }, - "5167daffe92e44d2acc2af2d9b9738df": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 6 ---------------------------------------------\n", + "[[Mainland china politics (80%)]] --> [[[SKIPPED]]]\n", + "\n", + "朴槿惠今访华 韩媒称访西安可能为增进与习近平友谊\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 2 / 6: 35%|███▌ | 7/20 [15:04<27:59, 129.16s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 3 / 7: 35%|███▌ | 7/20 [15:04<27:59, 129.16s/it]\u001b[A" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 7 ---------------------------------------------\n", + "[[Mainland china politics (59%)]] --> [[[SKIPPED]]]\n", + "\n", + "中国驻休斯敦总领馆举办春节招待会向华裔拜年\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 3 / 7: 40%|████ | 8/20 [15:08<22:43, 113.60s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 4 / 8: 40%|████ | 8/20 [15:08<22:43, 113.61s/it]\u001b[A" + ] }, - "acbfb34a353f41649675bd104069d14e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 8 ---------------------------------------------\n", + "[[Culture (93%)]] --> [[[SKIPPED]]]\n", + "\n", + "NASA发现“地球兄弟” 具备生命存活条件\n", + "\n", + "\n" + ] }, - "be070cb4a1624b0bb8f9b594c6b951a5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_2edb7130713d4e10a07bbf808abb9771", - "IPY_MODEL_5ae4c618f75d4ef9b65e5020fccb6d72", - "IPY_MODEL_138d8260e67f4bc58106b9b42f7abd12" - ], - "layout": "IPY_MODEL_d7621b5c619a4ce38ebe63924374cf78" - } + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 4 / 8: 45%|████▌ | 9/20 [15:13<18:36, 101.52s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 5 / 9: 45%|████▌ | 9/20 [15:13<18:36, 101.52s/it]\u001b[A" + ] }, - "2edb7130713d4e10a07bbf808abb9771": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1b208b6df75f4a9e97faa4e3705a9442", - "placeholder": "​", - "style": "IPY_MODEL_a7871b8ec3ec40e7bbbe6a5f40b79f4a", - "value": "Downloading: 100%" - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 9 ---------------------------------------------\n", + "[[Culture (53%)]] --> [[[SKIPPED]]]\n", + "\n", + "儿子去世后社交网站账号停用 父亲请求保留记忆\n", + "\n", + "\n" + ] }, - "5ae4c618f75d4ef9b65e5020fccb6d72": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_aeb7ee752d834b4cbaa189419fd75dd4", - "max": 9096718, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b47dfff73e73410aa89f65e3c5b0c366", - "value": 9096718 - } + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 1 / 3 / 5 / 9: 50%|█████ | 10/20 [18:20<18:20, 110.06s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 2 / 3 / 5 / 10: 50%|█████ | 10/20 [18:20<18:20, 110.06s/it]\u001b[A" + ] }, - "138d8260e67f4bc58106b9b42f7abd12": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_bdf3571e59ef4a688ab89d4badda27b1", - "placeholder": "​", - "style": "IPY_MODEL_d3bab427b92144d6b9ce96eac18ceb89", - "value": " 8.68M/8.68M [00:00<00:00, 16.8MB/s]" - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 10 ---------------------------------------------\n", + "[[Culture (100%)]] --> [[Entertainment (72%)]]\n", + "\n", + "第六届鲁迅文学奖颁发 格非等35位获奖者领奖\n", + "\n", + "第六届决赛颁发 格非等35位获奖者领奖\n", + "\n", + "\n" + ] }, - "d7621b5c619a4ce38ebe63924374cf78": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 2 / 3 / 5 / 10: 55%|█████▌ | 11/20 [22:44<18:36, 124.02s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 3 / 3 / 5 / 11: 55%|█████▌ | 11/20 [22:44<18:36, 124.02s/it]\u001b[A" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 11 ---------------------------------------------\n", + "[[Hong kong - macau politics (96%)]] --> [[Culture (79%)]]\n", + "\n", + "东莞台商欲借“台博会”搭建内销平台\n", + "\n", + "东莞讯欲借“艺博会”搭建内销平台\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 3 / 3 / 5 / 11: 60%|██████ | 12/20 [22:48<15:12, 114.07s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 3 / 3 / 6 / 12: 60%|██████ | 12/20 [22:48<15:12, 114.07s/it]\u001b[A" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 12 ---------------------------------------------\n", + "[[Financial news (56%)]] --> [[[SKIPPED]]]\n", + "\n", + "日本网友买扇贝当下酒菜 发现内有真正珍珠(图)\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 3 / 3 / 6 / 12: 65%|██████▌ | 13/20 [28:59<15:36, 133.78s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 3 / 4 / 6 / 13: 65%|██████▌ | 13/20 [28:59<15:36, 133.78s/it]\u001b[A" + ] }, - "1b208b6df75f4a9e97faa4e3705a9442": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 13 ---------------------------------------------\n", + "[[Sports (100%)]] --> [[[FAILED]]]\n", + "\n", + "篮球热潮席卷张江 NBA中投王与拉拉队鼎力加盟\n", + "\n", + "\n" + ] }, - "a7871b8ec3ec40e7bbbe6a5f40b79f4a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 3 / 4 / 6 / 13: 70%|███████ | 14/20 [33:40<14:26, 144.34s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 3 / 5 / 6 / 14: 70%|███████ | 14/20 [33:40<14:26, 144.34s/it]\u001b[A" + ] }, - "aeb7ee752d834b4cbaa189419fd75dd4": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 14 ---------------------------------------------\n", + "[[Sports (100%)]] --> [[[FAILED]]]\n", + "\n", + "UFC终极格斗冠军赛开打 \"草原狼\"遭遇三连败\n", + "\n", + "\n" + ] }, - "b47dfff73e73410aa89f65e3c5b0c366": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 3 / 5 / 6 / 14: 75%|███████▌ | 15/20 [33:45<11:15, 135.04s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 3 / 5 / 7 / 15: 75%|███████▌ | 15/20 [33:45<11:15, 135.04s/it]\u001b[A" + ] }, - "bdf3571e59ef4a688ab89d4badda27b1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 15 ---------------------------------------------\n", + "[[Culture (92%)]] --> [[[SKIPPED]]]\n", + "\n", + "水果style：心形水果惹人爱 骰子西瓜乐趣多(图)\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 3 / 5 / 7 / 15: 80%|████████ | 16/20 [40:09<10:02, 150.60s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 3 / 6 / 7 / 16: 80%|████████ | 16/20 [40:09<10:02, 150.60s/it]\u001b[A" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 16 ---------------------------------------------\n", + "[[Sports (100%)]] --> [[[FAILED]]]\n", + "\n", + "同里杯中国天元赛前瞻：芈昱廷李钦诚争挑战权\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 3 / 6 / 7 / 16: 85%|████████▌ | 17/20 [43:32<07:41, 153.67s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 4 / 6 / 7 / 17: 85%|████████▌ | 17/20 [43:32<07:41, 153.67s/it]\u001b[A" + ] }, - "d3bab427b92144d6b9ce96eac18ceb89": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - } - } - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "m83IiqVREJ96" - }, - "source": [ - "# Chinese Attack" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6UZ0d84hEJ98" - }, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/QData/TextAttack/blob/master/docs/2notebook/Example_6_Chinese%20Attack.ipynb)\n", - "\n", - "\n", - "[![View Source on GitHub](https://img.shields.io/badge/github-view%20source-black.svg)](https://github.com/QData/TextAttack/blob/master/docs/2notebook/Example_6_Chinese%20Attack.ipynb)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tjqc2c5_7YaX" - }, - "source": [ - " Please remember to run the following in your notebook enviroment before running the tutorial codes:\n", - "\n", - "```\n", - "pip3 install textattack[tensorflow]\n", - "```\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qZ5xnoevEJ99" - }, - "source": [ - "With a few additional modifications to the standard TextAttack commands, lanaguage models in Chinese can be attacked just as English models. Four transformations are available for either Chinese attack or augmentation:\n", - "\n", - "1. **ChineseHomophoneCharacterSwap**: transforms an input by replacing its words with substitions that share similar/identical pronounciation.\n", - "2. **ChineseMorphonymCharacterSwap**: transforms an input by replacing its words with substitions that share similar glyph structures.\n", - "3. **ChineseWordSwapHowNet**: transforms an input by replacing its words with synonyms provided by [OpenHownet](http://nlp.csai.tsinghua.edu.cn/).\n", - "4. **ChineseWordSwapMaskedLM**: transforms an input with potential replacements using a masked language model." - ] - }, - { - "cell_type": "markdown", - "source": [ - "We begin with imports:" - ], - "metadata": { - "id": "2EP1DJylSfkD" - } - }, - { - "cell_type": "code", - "metadata": { - "id": "5AXyxiLD4X93" - }, - "source": [ - "# Import required packages\n", - "import transformers\n", - "import string\n", - "import os\n", - "import pandas as pd\n", - "import datasets\n", - "\n", - "# Import classes required to build an Attacker\n", - "from textattack.models.wrappers import HuggingFaceModelWrapper\n", - "from textattack.search_methods import GreedyWordSwapWIR\n", - "from textattack.constraints.pre_transformation import (\n", - " RepeatModification,\n", - " StopwordModification,\n", - ")\n", - "from textattack.goal_functions import UntargetedClassification\n", - "\n", - "from textattack import Attack, Attacker, AttackArgs\n", - "from textattack.loggers import CSVLogger\n", - "from textattack.datasets import Dataset, HuggingFaceDataset\n", - "\n", - "# Import optional MUSE for higher quality examples\n", - "from textattack.constraints.semantics.sentence_encoders import (\n", - " MultilingualUniversalSentenceEncoder,\n", - ")\n", - "\n", - "muse = MultilingualUniversalSentenceEncoder(\n", - " threshold=0.9,\n", - " metric=\"cosine\",\n", - " compare_against_original=True,\n", - " window_size=15,\n", - " skip_text_shorter_than_window=True,\n", - ")\n", - "\n", - "# Import the transformations\n", - "\n", - "from textattack.transformations import CompositeTransformation\n", - "from textattack.transformations import ChineseWordSwapMaskedLM\n", - "from textattack.transformations import ChineseMorphonymCharacterSwap\n", - "from textattack.transformations import ChineseWordSwapHowNet\n", - "from textattack.transformations import ChineseHomophoneCharacterSwap" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "Models and datasets would also need to be set up:" - ], - "metadata": { - "id": "1mSvCqhHSi0h" - } - }, - { - "cell_type": "code", - "source": [ - "# In this example, we will attack a pre-trained entailment model from HugginFace (https://huggingface.co/uer/roberta-base-finetuned-chinanews-chinese)\n", - "tokenizer = transformers.AutoTokenizer.from_pretrained(\n", - " \"uer/roberta-base-finetuned-chinanews-chinese\"\n", - ")\n", - "model = transformers.AutoModelForSequenceClassification.from_pretrained(\n", - " \"uer/roberta-base-finetuned-chinanews-chinese\"\n", - ")\n", - "model_wrapper = HuggingFaceModelWrapper(model, tokenizer)\n", - "\n", - "# Set goal function\n", - "goal_function = UntargetedClassification(model_wrapper, query_budget=10000)\n", - "\n", - "# Set dataset from which we will generate adversraial examples\n", - "path = os.path.abspath(\"\")\n", - "path_list = path.split(os.sep)\n", - "temppath = os.path.normpath(\"examples/dataset/zh_sentiment/entailment_dataset.tsv\")\n", - "dataset = datasets.load_dataset(\"csv\", data_files=temppath, delimiter=\"\\t\")[\"train\"]\n", - "dataset = HuggingFaceDataset(\n", - " dataset,\n", - " dataset_columns=([\"text\"], \"label\"),\n", - " label_names=[\n", - " \"Mainland China politics\",\n", - " \"Hong Kong - Macau politics\",\n", - " \"International news\",\n", - " \"Financial news\",\n", - " \"Culture\",\n", - " \"Entertainment\",\n", - " \"Sports\",\n", - " ],\n", - ")" - ], - "metadata": { - "id": "CfnC9qUFPq9h" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "If this is your first time running Hownet, run this code block" - ], - "metadata": { - "id": "XfJVzCdRSr3d" - } - }, - { - "cell_type": "code", - "source": [ - "import OpenHowNet\n", - "\n", - "OpenHowNet.download()" - ], - "metadata": { - "id": "Hgal-PHeQwys" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "\n", - "\n", - "Now we are ready to attack! With goal function, transformation, constraints, search method, and goal function, we create the Attacker as any other TextAttack attacks\n" + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 17 ---------------------------------------------\n", + "[[Entertainment (100%)]] --> [[Financial news (99%)]]\n", + "\n", + "桂纶镁为戏体验生活 东北洗衣店当店员\n", + "\n", + "桂纶品牌为首体验生活 东北洗衣店当家\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 4 / 6 / 7 / 17: 90%|█████████ | 18/20 [44:01<04:53, 146.75s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 4 / 7 / 7 / 18: 90%|█████████ | 18/20 [44:01<04:53, 146.75s/it]\u001b[A" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 18 ---------------------------------------------\n", + "[[Culture (95%)]] --> [[[FAILED]]]\n", + "\n", + "河南羲皇故都朝祖会流传6000年 一天游客80万人\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 4 / 7 / 7 / 18: 95%|█████████▌| 19/20 [44:07<02:19, 139.35s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 4 / 7 / 8 / 19: 95%|█████████▌| 19/20 [44:07<02:19, 139.35s/it]\u001b[A" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 19 ---------------------------------------------\n", + "[[Culture (92%)]] --> [[[SKIPPED]]]\n", + "\n", + "辛柏青谈追求妻子：用1袋洗衣粉、2块肥皂打动她的\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[Succeeded / Failed / Skipped / Total] 4 / 7 / 8 / 19: 100%|██████████| 20/20 [49:19<00:00, 147.96s/it]\u001b[A\n", + "[Succeeded / Failed / Skipped / Total] 5 / 7 / 8 / 20: 100%|██████████| 20/20 [49:19<00:00, 147.96s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------- Result 20 ---------------------------------------------\n", + "[[International news (100%)]] --> [[Mainland china politics (66%)]]\n", + "\n", + "朝鲜谴责韩国前方部队打出反朝口号\n", + "\n", + "中国谴责日本前方部队打出侵略口号\n", + "\n", + "\n", + "\n", + "+-------------------------------+--------+\n", + "| Attack Results | |\n", + "+-------------------------------+--------+\n", + "| Number of successful attacks: | 5 |\n", + "| Number of failed attacks: | 7 |\n", + "| Number of skipped attacks: | 8 |\n", + "| Original accuracy: | 60.0% |\n", + "| Accuracy under attack: | 35.0% |\n", + "| Attack success rate: | 41.67% |\n", + "| Average perturbed word %: | 36.39% |\n", + "| Average num. words per input: | 9.3 |\n", + "| Avg num queries: | 45.5 |\n", + "+-------------------------------+--------+\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } ], - "metadata": { - "id": "SrtoxdrMSZ0X" - } - }, - { - "cell_type": "code", "source": [ "# transformation, using ChineseWordSwapMaskedLM transformation in this example\n", "\n", @@ -2405,669 +1605,1469 @@ "attack_args = AttackArgs(num_examples=20)\n", "attacker = Attacker(attack, dataset, attack_args)\n", "attack_results = attacker.attack_dataset()" - ], + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3e_tQiHWS-Pb" + }, + "source": [ + "As aforementioned, we can also augment Chinese sentences with the provided transformation. A quick examples is shown below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": { "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000, - "referenced_widgets": [ - "4b423038915e40158f9da4c07d09aad3", - "3711cf0a18994cee8fc840d9a93cf5d3", - "7f77bd7b8e5f45ae94cfc45f915c0c72", - "fe0ca6138bc54b628c03e590c6e96aed", - "8b39363f69eb46009c5357263a65248c", - "6b976fd913584da69456c1b6d53483cb", - "ea568ab2407f474da3b1f1b2540fa3a8", - "ff6b34a7e75b443593f3dca5d050cd52", - "4f31972fd2fd44bbac063bb4b5075e98", - "7de1551891ec447ab6d80ea1de145f16", - "e5e2c0507c834887b80f5717c1e6d5f3", - "588b1321a9274de6a8a9e86622d90be4", - "2436b07259a34ee18fe9c1007f7b615b", - "98aac5a0baee4930bd461f2c5fd73f4a", - "34607a8556794a5a86c18abe5bd7e5a5", - "f78f6701ce4f4b3b9ff0af925620f261", - "a1e3fb5cceed4e95957a17192a641b69", - "83e9b14c4d354fdc80db4f8a881f19f3", - "5f5457f292284dd8b914f45e26b2f749", - "2bb72191846f49528663680a315d8b01", - "83eff532314e4edcbfe648b321e9a310", - "3d30e700d32443fdb37b5ab934d2d70a", - "a132f09845a54cbe865cbe8159bb693e", - "0af0e1eaea2f48c5b0fec6e550bd1baa", - "dd6b0a5d9db245338a8fdb2ef5b29bf9", - "58fc309041b54e94ae265167fa20d8d7", - "89dfd3fdc41e417a870901bc79e47495", - "21472d1c4c8b494a8d3660b3320e9d4b", - "7511bb9ca5424674bb2350dff63c468a", - "f6dd2c2cb4e346fe9af7026b5d2162e9", - "a34ad57624fc422aa4832db3963298e6", - "5167daffe92e44d2acc2af2d9b9738df", - "acbfb34a353f41649675bd104069d14e", - "be070cb4a1624b0bb8f9b594c6b951a5", - "2edb7130713d4e10a07bbf808abb9771", - "5ae4c618f75d4ef9b65e5020fccb6d72", - "138d8260e67f4bc58106b9b42f7abd12", - "d7621b5c619a4ce38ebe63924374cf78", - "1b208b6df75f4a9e97faa4e3705a9442", - "a7871b8ec3ec40e7bbbe6a5f40b79f4a", - "aeb7ee752d834b4cbaa189419fd75dd4", - "b47dfff73e73410aa89f65e3c5b0c366", - "bdf3571e59ef4a688ab89d4badda27b1", - "d3bab427b92144d6b9ce96eac18ceb89" - ] + "base_uri": "https://localhost:8080/" }, - "id": "C_0Z8njnRblT", - "outputId": "3890d784-de7f-4b70-f984-cbc9e0c7f700" + "id": "43MCRE0pqVM0", + "outputId": "2ad12bf5-3bd8-4c8d-913c-949fcae787d3" }, - "execution_count": null, "outputs": [ { - "output_type": "display_data", - "data": { - "text/plain": [ - "Downloading: 0%| | 0.00/615 [00:00 [[[FAILED]]]\n", - "\n", - "林书豪新秀赛上甘心\"跑龙套\" 自称仍是底薪球员\n", - "\n", - "\n" - ] + "1b208b6df75f4a9e97faa4e3705a9442": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 0 / 1 / 0 / 1: 10%|█ | 2/20 [06:55<1:02:18, 207.69s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 0 / 2 / 0 / 2: 10%|█ | 2/20 [06:55<1:02:18, 207.70s/it]\u001b[A" - ] + "21472d1c4c8b494a8d3660b3320e9d4b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 2 ---------------------------------------------\n", - "[[Culture (100%)]] --> [[[FAILED]]]\n", - "\n", - "成都现“真人图书馆”：无书“借人”给你读\n", - "\n", - "\n" - ] + "2436b07259a34ee18fe9c1007f7b615b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a1e3fb5cceed4e95957a17192a641b69", + "placeholder": "​", + "style": "IPY_MODEL_83e9b14c4d354fdc80db4f8a881f19f3", + "value": "Downloading: 100%" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 0 / 2 / 0 / 2: 15%|█▌ | 3/20 [07:01<39:50, 140.61s/it] \u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 0 / 2 / 1 / 3: 15%|█▌ | 3/20 [07:01<39:50, 140.61s/it]\u001b[A" - ] + "2bb72191846f49528663680a315d8b01": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 3 ---------------------------------------------\n", - "[[Mainland china politics (57%)]] --> [[[SKIPPED]]]\n", - "\n", - "中国经济走向更趋稳健务实\n", - "\n", - "\n" - ] + "2edb7130713d4e10a07bbf808abb9771": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1b208b6df75f4a9e97faa4e3705a9442", + "placeholder": "​", + "style": "IPY_MODEL_a7871b8ec3ec40e7bbbe6a5f40b79f4a", + "value": "Downloading: 100%" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 0 / 2 / 1 / 3: 20%|██ | 4/20 [11:33<46:12, 173.28s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 0 / 3 / 1 / 4: 20%|██ | 4/20 [11:33<46:12, 173.28s/it]\u001b[A" - ] + "34607a8556794a5a86c18abe5bd7e5a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_83eff532314e4edcbfe648b321e9a310", + "placeholder": "​", + "style": "IPY_MODEL_3d30e700d32443fdb37b5ab934d2d70a", + "value": " 1.04G/1.04G [00:25<00:00, 45.4MB/s]" + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 4 ---------------------------------------------\n", - "[[Sports (100%)]] --> [[[FAILED]]]\n", - "\n", - "国际田联世界挑战赛 罗伯斯迎来赛季第三冠\n", - "\n", - "\n" - ] + "3711cf0a18994cee8fc840d9a93cf5d3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6b976fd913584da69456c1b6d53483cb", + "placeholder": "​", + "style": "IPY_MODEL_ea568ab2407f474da3b1f1b2540fa3a8", + "value": "Downloading: 100%" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 0 / 3 / 1 / 4: 25%|██▌ | 5/20 [14:52<44:36, 178.44s/it]\u001b[A" - ] + "3d30e700d32443fdb37b5ab934d2d70a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 5 ---------------------------------------------\n" - ] + "4b423038915e40158f9da4c07d09aad3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_3711cf0a18994cee8fc840d9a93cf5d3", + "IPY_MODEL_7f77bd7b8e5f45ae94cfc45f915c0c72", + "IPY_MODEL_fe0ca6138bc54b628c03e590c6e96aed" + ], + "layout": "IPY_MODEL_8b39363f69eb46009c5357263a65248c" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 1 / 5: 25%|██▌ | 5/20 [14:53<44:39, 178.62s/it]\u001b[A" - ] + "4f31972fd2fd44bbac063bb4b5075e98": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "[[International news (66%)]] --> [[Entertainment (68%)]]\n", - "\n", - "德国一电视台合成“默克尔头巾照”惹争议\n", - "\n", - "德国一电视台合成“性感头巾照”惹争议\n", - "\n", - "\n" - ] + "5167daffe92e44d2acc2af2d9b9738df": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 1 / 5: 30%|███ | 6/20 [14:57<34:55, 149.65s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 2 / 6: 30%|███ | 6/20 [14:57<34:55, 149.65s/it]\u001b[A" - ] + "588b1321a9274de6a8a9e86622d90be4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_2436b07259a34ee18fe9c1007f7b615b", + "IPY_MODEL_98aac5a0baee4930bd461f2c5fd73f4a", + "IPY_MODEL_34607a8556794a5a86c18abe5bd7e5a5" + ], + "layout": "IPY_MODEL_f78f6701ce4f4b3b9ff0af925620f261" + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 6 ---------------------------------------------\n", - "[[Mainland china politics (80%)]] --> [[[SKIPPED]]]\n", - "\n", - "朴槿惠今访华 韩媒称访西安可能为增进与习近平友谊\n", - "\n", - "\n" - ] + "58fc309041b54e94ae265167fa20d8d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5167daffe92e44d2acc2af2d9b9738df", + "placeholder": "​", + "style": "IPY_MODEL_acbfb34a353f41649675bd104069d14e", + "value": " 4.83M/4.83M [00:00<00:00, 12.1MB/s]" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 2 / 6: 35%|███▌ | 7/20 [15:04<27:59, 129.16s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 3 / 7: 35%|███▌ | 7/20 [15:04<27:59, 129.16s/it]\u001b[A" - ] + "5ae4c618f75d4ef9b65e5020fccb6d72": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_aeb7ee752d834b4cbaa189419fd75dd4", + "max": 9096718, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_b47dfff73e73410aa89f65e3c5b0c366", + "value": 9096718 + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 7 ---------------------------------------------\n", - "[[Mainland china politics (59%)]] --> [[[SKIPPED]]]\n", - "\n", - "中国驻休斯敦总领馆举办春节招待会向华裔拜年\n", - "\n", - "\n" - ] + "5f5457f292284dd8b914f45e26b2f749": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 3 / 7: 40%|████ | 8/20 [15:08<22:43, 113.60s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 4 / 8: 40%|████ | 8/20 [15:08<22:43, 113.61s/it]\u001b[A" - ] + "6b976fd913584da69456c1b6d53483cb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 8 ---------------------------------------------\n", - "[[Culture (93%)]] --> [[[SKIPPED]]]\n", - "\n", - "NASA发现“地球兄弟” 具备生命存活条件\n", - "\n", - "\n" - ] + "7511bb9ca5424674bb2350dff63c468a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 4 / 8: 45%|████▌ | 9/20 [15:13<18:36, 101.52s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 5 / 9: 45%|████▌ | 9/20 [15:13<18:36, 101.52s/it]\u001b[A" - ] + "7de1551891ec447ab6d80ea1de145f16": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 9 ---------------------------------------------\n", - "[[Culture (53%)]] --> [[[SKIPPED]]]\n", - "\n", - "儿子去世后社交网站账号停用 父亲请求保留记忆\n", - "\n", - "\n" - ] + "7f77bd7b8e5f45ae94cfc45f915c0c72": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ff6b34a7e75b443593f3dca5d050cd52", + "max": 615, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4f31972fd2fd44bbac063bb4b5075e98", + "value": 615 + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 1 / 3 / 5 / 9: 50%|█████ | 10/20 [18:20<18:20, 110.06s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 2 / 3 / 5 / 10: 50%|█████ | 10/20 [18:20<18:20, 110.06s/it]\u001b[A" - ] + "83e9b14c4d354fdc80db4f8a881f19f3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 10 ---------------------------------------------\n", - "[[Culture (100%)]] --> [[Entertainment (72%)]]\n", - "\n", - "第六届鲁迅文学奖颁发 格非等35位获奖者领奖\n", - "\n", - "第六届决赛颁发 格非等35位获奖者领奖\n", - "\n", - "\n" - ] + "83eff532314e4edcbfe648b321e9a310": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 2 / 3 / 5 / 10: 55%|█████▌ | 11/20 [22:44<18:36, 124.02s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 3 / 3 / 5 / 11: 55%|█████▌ | 11/20 [22:44<18:36, 124.02s/it]\u001b[A" - ] + "89dfd3fdc41e417a870901bc79e47495": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 11 ---------------------------------------------\n", - "[[Hong kong - macau politics (96%)]] --> [[Culture (79%)]]\n", - "\n", - "东莞台商欲借“台博会”搭建内销平台\n", - "\n", - "东莞讯欲借“艺博会”搭建内销平台\n", - "\n", - "\n" - ] + "8b39363f69eb46009c5357263a65248c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 3 / 3 / 5 / 11: 60%|██████ | 12/20 [22:48<15:12, 114.07s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 3 / 3 / 6 / 12: 60%|██████ | 12/20 [22:48<15:12, 114.07s/it]\u001b[A" - ] + "98aac5a0baee4930bd461f2c5fd73f4a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5f5457f292284dd8b914f45e26b2f749", + "max": 1115590446, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2bb72191846f49528663680a315d8b01", + "value": 1115590446 + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 12 ---------------------------------------------\n", - "[[Financial news (56%)]] --> [[[SKIPPED]]]\n", - "\n", - "日本网友买扇贝当下酒菜 发现内有真正珍珠(图)\n", - "\n", - "\n" - ] + "a132f09845a54cbe865cbe8159bb693e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_0af0e1eaea2f48c5b0fec6e550bd1baa", + "IPY_MODEL_dd6b0a5d9db245338a8fdb2ef5b29bf9", + "IPY_MODEL_58fc309041b54e94ae265167fa20d8d7" + ], + "layout": "IPY_MODEL_89dfd3fdc41e417a870901bc79e47495" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 3 / 3 / 6 / 12: 65%|██████▌ | 13/20 [28:59<15:36, 133.78s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 3 / 4 / 6 / 13: 65%|██████▌ | 13/20 [28:59<15:36, 133.78s/it]\u001b[A" - ] + "a1e3fb5cceed4e95957a17192a641b69": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 13 ---------------------------------------------\n", - "[[Sports (100%)]] --> [[[FAILED]]]\n", - "\n", - "篮球热潮席卷张江 NBA中投王与拉拉队鼎力加盟\n", - "\n", - "\n" - ] + "a34ad57624fc422aa4832db3963298e6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 3 / 4 / 6 / 13: 70%|███████ | 14/20 [33:40<14:26, 144.34s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 3 / 5 / 6 / 14: 70%|███████ | 14/20 [33:40<14:26, 144.34s/it]\u001b[A" - ] + "a7871b8ec3ec40e7bbbe6a5f40b79f4a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 14 ---------------------------------------------\n", - "[[Sports (100%)]] --> [[[FAILED]]]\n", - "\n", - "UFC终极格斗冠军赛开打 \"草原狼\"遭遇三连败\n", - "\n", - "\n" - ] + "acbfb34a353f41649675bd104069d14e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 3 / 5 / 6 / 14: 75%|███████▌ | 15/20 [33:45<11:15, 135.04s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 3 / 5 / 7 / 15: 75%|███████▌ | 15/20 [33:45<11:15, 135.04s/it]\u001b[A" - ] + "aeb7ee752d834b4cbaa189419fd75dd4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 15 ---------------------------------------------\n", - "[[Culture (92%)]] --> [[[SKIPPED]]]\n", - "\n", - "水果style：心形水果惹人爱 骰子西瓜乐趣多(图)\n", - "\n", - "\n" - ] + "b47dfff73e73410aa89f65e3c5b0c366": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 3 / 5 / 7 / 15: 80%|████████ | 16/20 [40:09<10:02, 150.60s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 3 / 6 / 7 / 16: 80%|████████ | 16/20 [40:09<10:02, 150.60s/it]\u001b[A" - ] + "bdf3571e59ef4a688ab89d4badda27b1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 16 ---------------------------------------------\n", - "[[Sports (100%)]] --> [[[FAILED]]]\n", - "\n", - "同里杯中国天元赛前瞻：芈昱廷李钦诚争挑战权\n", - "\n", - "\n" - ] + "be070cb4a1624b0bb8f9b594c6b951a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_2edb7130713d4e10a07bbf808abb9771", + "IPY_MODEL_5ae4c618f75d4ef9b65e5020fccb6d72", + "IPY_MODEL_138d8260e67f4bc58106b9b42f7abd12" + ], + "layout": "IPY_MODEL_d7621b5c619a4ce38ebe63924374cf78" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 3 / 6 / 7 / 16: 85%|████████▌ | 17/20 [43:32<07:41, 153.67s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 4 / 6 / 7 / 17: 85%|████████▌ | 17/20 [43:32<07:41, 153.67s/it]\u001b[A" - ] + "d3bab427b92144d6b9ce96eac18ceb89": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 17 ---------------------------------------------\n", - "[[Entertainment (100%)]] --> [[Financial news (99%)]]\n", - "\n", - "桂纶镁为戏体验生活 东北洗衣店当店员\n", - "\n", - "桂纶品牌为首体验生活 东北洗衣店当家\n", - "\n", - "\n" - ] + "d7621b5c619a4ce38ebe63924374cf78": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 4 / 6 / 7 / 17: 90%|█████████ | 18/20 [44:01<04:53, 146.75s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 4 / 7 / 7 / 18: 90%|█████████ | 18/20 [44:01<04:53, 146.75s/it]\u001b[A" - ] + "dd6b0a5d9db245338a8fdb2ef5b29bf9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f6dd2c2cb4e346fe9af7026b5d2162e9", + "max": 5069051, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a34ad57624fc422aa4832db3963298e6", + "value": 5069051 + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 18 ---------------------------------------------\n", - "[[Culture (95%)]] --> [[[FAILED]]]\n", - "\n", - "河南羲皇故都朝祖会流传6000年 一天游客80万人\n", - "\n", - "\n" - ] + "e5e2c0507c834887b80f5717c1e6d5f3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 4 / 7 / 7 / 18: 95%|█████████▌| 19/20 [44:07<02:19, 139.35s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 4 / 7 / 8 / 19: 95%|█████████▌| 19/20 [44:07<02:19, 139.35s/it]\u001b[A" - ] + "ea568ab2407f474da3b1f1b2540fa3a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 19 ---------------------------------------------\n", - "[[Culture (92%)]] --> [[[SKIPPED]]]\n", - "\n", - "辛柏青谈追求妻子：用1袋洗衣粉、2块肥皂打动她的\n", - "\n", - "\n" - ] + "f6dd2c2cb4e346fe9af7026b5d2162e9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n", - "[Succeeded / Failed / Skipped / Total] 4 / 7 / 8 / 19: 100%|██████████| 20/20 [49:19<00:00, 147.96s/it]\u001b[A\n", - "[Succeeded / Failed / Skipped / Total] 5 / 7 / 8 / 20: 100%|██████████| 20/20 [49:19<00:00, 147.96s/it]" - ] + "f78f6701ce4f4b3b9ff0af925620f261": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "--------------------------------------------- Result 20 ---------------------------------------------\n", - "[[International news (100%)]] --> [[Mainland china politics (66%)]]\n", - "\n", - "朝鲜谴责韩国前方部队打出反朝口号\n", - "\n", - "中国谴责日本前方部队打出侵略口号\n", - "\n", - "\n", - "\n", - "+-------------------------------+--------+\n", - "| Attack Results | |\n", - "+-------------------------------+--------+\n", - "| Number of successful attacks: | 5 |\n", - "| Number of failed attacks: | 7 |\n", - "| Number of skipped attacks: | 8 |\n", - "| Original accuracy: | 60.0% |\n", - "| Accuracy under attack: | 35.0% |\n", - "| Attack success rate: | 41.67% |\n", - "| Average perturbed word %: | 36.39% |\n", - "| Average num. words per input: | 9.3 |\n", - "| Avg num queries: | 45.5 |\n", - "+-------------------------------+--------+\n" - ] + "fe0ca6138bc54b628c03e590c6e96aed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7de1551891ec447ab6d80ea1de145f16", + "placeholder": "​", + "style": "IPY_MODEL_e5e2c0507c834887b80f5717c1e6d5f3", + "value": " 615/615 [00:00<00:00, 33.8kB/s]" + } }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "\n" - ] + "ff6b34a7e75b443593f3dca5d050cd52": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } } - ] - }, - { - "cell_type": "markdown", - "source": [ - "As aforementioned, we can also augment Chinese sentences with the provided transformation. A quick examples is shown below:" - ], - "metadata": { - "id": "3e_tQiHWS-Pb" } - }, - { - "cell_type": "code", - "source": [ - "from textattack.constraints.pre_transformation import RepeatModification\n", - "from textattack.constraints.pre_transformation import StopwordModification\n", - "from textattack.augmentation import Augmenter\n", - "\n", - "# transformation\n", - "transformation = ChineseMorphonymCharacterSwap()\n", - "\n", - "# constraints\n", - "constraints = [RepeatModification(), StopwordModification()]\n", - "\n", - "# Create augmenter with specified parameters\n", - "augmenter = Augmenter(\n", - " transformation=transformation, pct_words_to_swap=0.1, transformations_per_example=2\n", - ")\n", - "s = \"听见树林的呢喃，发现溪流中的知识。\"\n", - "\n", - "# Augment!\n", - "augmenter.augment(s)" - ], - "metadata": { - "id": "43MCRE0pqVM0", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "2ad12bf5-3bd8-4c8d-913c-949fcae787d3" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "Building prefix dict from the default dictionary ...\n", - "DEBUG:jieba:Building prefix dict from the default dictionary ...\n", - "Dumping model to file cache /tmp/jieba.cache\n", - "DEBUG:jieba:Dumping model to file cache /tmp/jieba.cache\n", - "Loading model cost 0.888 seconds.\n", - "DEBUG:jieba:Loading model cost 0.888 seconds.\n", - "Prefix dict has been built successfully.\n", - "DEBUG:jieba:Prefix dict has been built successfully.\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "['听见树林的呢喃，发现溪流中的知织。', '听见树林的呢喃，发视溪流中的知识。']" - ] - }, - "metadata": {}, - "execution_count": 11 - } - ] } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/tests/test_attacked_text.py b/tests/test_attacked_text.py index 6aff12fbc..50bdf86b4 100644 --- a/tests/test_attacked_text.py +++ b/tests/test_attacked_text.py @@ -70,7 +70,7 @@ def test_window_around_index(self, attacked_text): def test_big_window_around_index(self, attacked_text): assert ( - attacked_text.text_window_around_index(0, 10**5) + "." + attacked_text.text_window_around_index(0, 10 ** 5) + "." ) == attacked_text.text def test_window_around_index_start(self, attacked_text): diff --git a/tests/test_command_line/test_loggers.py b/tests/test_command_line/test_loggers.py new file mode 100644 index 000000000..c6589f60a --- /dev/null +++ b/tests/test_command_line/test_loggers.py @@ -0,0 +1,101 @@ +import json +import os + +from helpers import run_command_and_get_result +import pytest + +DEBUG = False + +""" +Attack command-line tests in the format (name, args, sample_output_file) +""" + +""" + list_test_params data structure requires + 1) test name + 2) logger filetype - json/text/csv. # Future Work : Tests for Wandb and Visdom + 3) logger file name + 4) sample log file +""" + +list_test_params = [ + ( + "json_summary_logger", + "json", + "textattack attack --recipe deepwordbug --model lstm-mr --num-examples 2 --log-summary-to-json attack_summary.json", + "attack_summary.json", + "tests/sample_outputs/json_attack_summary.json", + ), + ( + "txt_logger", + "txt", + "textattack attack --recipe deepwordbug --model lstm-mr --num-examples 2 --log-to-txt attack_log.txt", + "attack_log.txt", + "tests/sample_outputs/txt_attack_log.txt", + ), + # Removing CSV Logging Test for time-being , will redo CSV test in separate PR. + # ( + # "csv_logger", + # "csv", + # "textattack attack --recipe deepwordbug --model lstm-mr --num-examples 2 --log-to-csv attack_log.csv", + # "attack_log.csv", + # "tests/sample_outputs/csv_attack_log.csv", + # ), +] + + +@pytest.mark.parametrize( + "name, filetype, command, test_log_file, sample_log_file", list_test_params +) +def test_logger(name, filetype, command, test_log_file, sample_log_file): + # Run command and validate outputs. + result = run_command_and_get_result(command) + + assert result.stdout is not None + assert result.stderr is not None + assert result.returncode == 0 + assert os.path.exists(test_log_file), f"{test_log_file} did not get generated" + + if filetype == "json": + with open(sample_log_file) as f: + desired_dictionary = json.load(f) + + with open(test_log_file) as f: + test_dictionary = json.load(f) + + assert ( + desired_dictionary == test_dictionary + ), f"{filetype} file {test_log_file} differs from {sample_log_file}" + + elif filetype == "txt": + assert ( + os.system(f"diff {test_log_file} {sample_log_file}") == 0 + ), f"{filetype} file {test_log_file} differs from {sample_log_file}" + + elif filetype == "csv": + import pandas as pd + + # Convert them into dataframes and compare. + test_df = pd.read_csv(test_log_file) + sample_df = pd.read_csv(sample_log_file) + try: + test_df = test_df[sorted(list(test_df.columns.values))] + sample_df = sample_df[sorted(list(test_df.columns.values))] + + for c in test_df.columns: + if test_df[c].dtype == int: + test_df[c] = test_df[c].astype(float) + + if sample_df[c].dtype == int: + sample_df[c] = sample_df[c].astype(float) + except KeyError: + assert ( + False + ), f"{filetype} file {test_log_file} differs from {sample_log_file}" + + assert sample_df.equals( + test_df + ), f"{filetype} file {test_log_file} differs from {sample_log_file}" + + # cleanup + os.remove(test_log_file) diff --git a/tests/test_word_embedding.py b/tests/test_word_embedding.py index 4772c27dd..5232e8fa1 100644 --- a/tests/test_word_embedding.py +++ b/tests/test_word_embedding.py @@ -10,7 +10,7 @@ def test_embedding_paragramcf(): word_embedding = WordEmbedding.counterfitted_GLOVE_embedding() assert pytest.approx(word_embedding[0][0]) == -0.022007 assert pytest.approx(word_embedding["fawn"][0]) == -0.022007 - assert word_embedding[10**9] is None + assert word_embedding[10 ** 9] is None def test_embedding_gensim(): @@ -37,7 +37,7 @@ def test_embedding_gensim(): word_embedding = GensimWordEmbedding(keyed_vectors) assert pytest.approx(word_embedding[0][0]) == 1 assert pytest.approx(word_embedding["bye-bye"][0]) == -1 / np.sqrt(2) - assert word_embedding[10**9] is None + assert word_embedding[10 ** 9] is None # test query functionality assert pytest.approx(word_embedding.get_cos_sim(1, 3)) == 0 diff --git a/textattack/attack.py b/textattack/attack.py index 47537d1b0..7743817ab 100644 --- a/textattack/attack.py +++ b/textattack/attack.py @@ -83,8 +83,8 @@ def __init__( constraints: List[Union[Constraint, PreTransformationConstraint]], transformation: Transformation, search_method: SearchMethod, - transformation_cache_size=2**15, - constraint_cache_size=2**15, + transformation_cache_size=2 ** 15, + constraint_cache_size=2 ** 15, ): """Initialize an attack object. diff --git a/textattack/attack_args.py b/textattack/attack_args.py index 0491ea17e..495e1006b 100644 --- a/textattack/attack_args.py +++ b/textattack/attack_args.py @@ -504,8 +504,8 @@ class _CommandLineAttackArgs: interactive: bool = False parallel: bool = False model_batch_size: int = 32 - model_cache_size: int = 2**18 - constraint_cache_size: int = 2**18 + model_cache_size: int = 2 ** 18 + constraint_cache_size: int = 2 ** 18 @classmethod def _add_parser_args(cls, parser): diff --git a/textattack/constraints/grammaticality/cola.py b/textattack/constraints/grammaticality/cola.py index 190bad25c..beb7c30a6 100644 --- a/textattack/constraints/grammaticality/cola.py +++ b/textattack/constraints/grammaticality/cola.py @@ -43,7 +43,7 @@ def __init__( self.max_diff = max_diff self.model_name = model_name - self._reference_score_cache = lru.LRU(2**10) + self._reference_score_cache = lru.LRU(2 ** 10) model = AutoModelForSequenceClassification.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) self.model = HuggingFaceModelWrapper(model, tokenizer) diff --git a/textattack/constraints/grammaticality/language_models/google_language_model/alzantot_goog_lm.py b/textattack/constraints/grammaticality/language_models/google_language_model/alzantot_goog_lm.py index 005dda55e..d47bfd6ec 100644 --- a/textattack/constraints/grammaticality/language_models/google_language_model/alzantot_goog_lm.py +++ b/textattack/constraints/grammaticality/language_models/google_language_model/alzantot_goog_lm.py @@ -49,7 +49,7 @@ def __init__(self): self.sess, self.graph, self.PBTXT_PATH, self.CKPT_PATH ) - self.lm_cache = lru.LRU(2**18) + self.lm_cache = lru.LRU(2 ** 18) def clear_cache(self): self.lm_cache.clear() diff --git a/textattack/constraints/grammaticality/part_of_speech.py b/textattack/constraints/grammaticality/part_of_speech.py index f531f33c7..e686efdb1 100644 --- a/textattack/constraints/grammaticality/part_of_speech.py +++ b/textattack/constraints/grammaticality/part_of_speech.py @@ -56,7 +56,7 @@ def __init__( self.language_nltk = language_nltk self.language_stanza = language_stanza - self._pos_tag_cache = lru.LRU(2**14) + self._pos_tag_cache = lru.LRU(2 ** 14) if tagger_type == "flair": if tagset == "universal": self._flair_pos_tagger = SequenceTagger.load("upos-fast") diff --git a/textattack/constraints/semantics/sentence_encoders/thought_vector.py b/textattack/constraints/semantics/sentence_encoders/thought_vector.py index 4a7978b01..60bac23ba 100644 --- a/textattack/constraints/semantics/sentence_encoders/thought_vector.py +++ b/textattack/constraints/semantics/sentence_encoders/thought_vector.py @@ -32,7 +32,7 @@ def __init__(self, embedding=None, **kwargs): def clear_cache(self): self._get_thought_vector.cache_clear() - @functools.lru_cache(maxsize=2**10) + @functools.lru_cache(maxsize=2 ** 10) def _get_thought_vector(self, text): """Sums the embeddings of all the words in ``text`` into a "thought vector".""" diff --git a/textattack/goal_functions/goal_function.py b/textattack/goal_functions/goal_function.py index 78693f670..7fa6a7c3e 100644 --- a/textattack/goal_functions/goal_function.py +++ b/textattack/goal_functions/goal_function.py @@ -40,7 +40,7 @@ def __init__( use_cache=True, query_budget=float("inf"), model_batch_size=32, - model_cache_size=2**20, + model_cache_size=2 ** 20, ): validators.validate_model_goal_function_compatibility( self.__class__, model_wrapper.model.__class__ @@ -176,7 +176,8 @@ def _call_model_uncached(self, attacked_text_list): if isinstance(batch_preds, list): outputs.extend(batch_preds) elif isinstance(batch_preds, np.ndarray): - outputs.append(batch_preds) + # outputs.append(batch_preds) + outputs.append(torch.tensor(batch_preds)) else: outputs.append(batch_preds) i += self.batch_size diff --git a/textattack/goal_functions/text/minimize_bleu.py b/textattack/goal_functions/text/minimize_bleu.py index 92613be5a..339995772 100644 --- a/textattack/goal_functions/text/minimize_bleu.py +++ b/textattack/goal_functions/text/minimize_bleu.py @@ -59,7 +59,7 @@ def extra_repr_keys(self): return ["maximizable", "target_bleu"] -@functools.lru_cache(maxsize=2**12) +@functools.lru_cache(maxsize=2 ** 12) def get_bleu(a, b): ref = a.words hyp = b.words diff --git a/textattack/goal_functions/text/non_overlapping_output.py b/textattack/goal_functions/text/non_overlapping_output.py index e2cb49820..443aa2366 100644 --- a/textattack/goal_functions/text/non_overlapping_output.py +++ b/textattack/goal_functions/text/non_overlapping_output.py @@ -38,12 +38,12 @@ def _get_score(self, model_output, _): return num_words_diff / len(get_words_cached(self.ground_truth_output)) -@functools.lru_cache(maxsize=2**12) +@functools.lru_cache(maxsize=2 ** 12) def get_words_cached(s): return np.array(words_from_text(s)) -@functools.lru_cache(maxsize=2**12) +@functools.lru_cache(maxsize=2 ** 12) def word_difference_score(s1, s2): """Returns the number of words that are non-overlapping between s1 and s2.""" diff --git a/textattack/metrics/attack_metrics/words_perturbed.py b/textattack/metrics/attack_metrics/words_perturbed.py index 6104de1b3..d4b128241 100644 --- a/textattack/metrics/attack_metrics/words_perturbed.py +++ b/textattack/metrics/attack_metrics/words_perturbed.py @@ -31,7 +31,7 @@ def calculate(self, results): self.total_attacks = len(self.results) self.all_num_words = np.zeros(len(self.results)) self.perturbed_word_percentages = np.zeros(len(self.results)) - self.num_words_changed_until_success = np.zeros(2**16) + self.num_words_changed_until_success = np.zeros(2 ** 16) self.max_words_changed = 0 for i, result in enumerate(self.results): From 987c926a91820e734d3ea5cef41d5791d060f31e Mon Sep 17 00:00:00 2001 From: Yanjun Qi Date: Mon, 11 Sep 2023 11:04:03 -0400 Subject: [PATCH 2/2] format update --- textattack/shared/validators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/textattack/shared/validators.py b/textattack/shared/validators.py index 4d9611d5a..55f4ed08c 100644 --- a/textattack/shared/validators.py +++ b/textattack/shared/validators.py @@ -4,6 +4,7 @@ Validators ensure compatibility between search methods, transformations, constraints, and goal functions. """ + import re import textattack