Merge pull request #202 from QData/repr

v0.2.0: repr improvements, other minor stuff

README.md

@@ -106,6 +106,7 @@ Attacks on classification tasks, like sentiment classification and entailment:
 - **faster-alzantot**: modified, faster version of the Alzantot et al. genetic algorithm, from (["Certified Robustness to Adversarial Word Substitutions" (Jia et al., 2019)](https://arxiv.org/abs/1909.00986)).
 - **deepwordbug**: Greedy replace-1 scoring and multi-transformation character-swap attack (["Black-box Generation of Adversarial Text Sequences to Evade Deep Learning Classifiers" (Gao et al., 2018)](https://arxiv.org/abs/1801.04354)).
 - **hotflip**: Beam search and gradient-based word swap (["HotFlip: White-Box Adversarial Examples for Text Classification" (Ebrahimi et al., 2017)](https://arxiv.org/abs/1712.06751)).
+- **iga**: Improved genetic algorithm attack from (["Natural Language Adversarial Attacks and Defenses in Word Level (Wang et al., 2019)"](https://arxiv.org/abs/1909.06723)
 - **input-reduction**: Reducing the input while maintaining the prediction through word importance ranking (["Pathologies of Neural Models Make Interpretation Difficult" (Feng et al., 2018)](https://arxiv.org/pdf/1804.07781.pdf)).
 - **kuleshov**: Greedy search and counterfitted embedding swap (["Adversarial Examples for Natural Language Classification Problems" (Kuleshov et al., 2018)](https://openreview.net/pdf?id=r1QZ3zbAZ)).
 - **pso**: Particle swarm optimization and HowNet synonym swap (["Word-level Textual Adversarial Attacking as Combinatorial Optimization" (Zang et al., 2020)](https://www.aclweb.org/anthology/2020.acl-main.540/)).

docs/attacks/attack_recipes.rst

@@ -41,8 +41,15 @@ HotFlip (HotFlip: White-Box Adversarial Examples for Text Classification)
 .. automodule:: textattack.attack_recipes.hotflip_ebrahimi_2017
    :members:
 
-Input Reduction
-################
+Improved Genetic Algorithm (Natural Language Adversarial Attacks and Defenses in Word Level)
+#################################################################################################
+
+.. automodule:: textattack.attack_recipes.iga_wang_2019
+   :members:
+
+
+Input Reduction (Pathologies of Neural Models Make Interpretations Difficult)
+####################################################################################
 
 .. automodule:: textattack.attack_recipes.input_reduction_feng_2018
    :members:
@@ -56,7 +63,7 @@ Kuleshov (Adversarial Examples for Natural Language Classification Problems)
 Particle Swarm Optimization (Word-level Textual Adversarial Attacking as Combinatorial Optimization)
 #####################################################################################################
 
-.. automodule:: textattack.attack_recipes.PSO_zang_2020
+.. automodule:: textattack.attack_recipes.pso_zang_2020
    :members:
 
 PWWS (Generating Natural Language Adversarial Examples through Probability Weighted Word Saliency)

docs/attacks/search_method.rst

@@ -34,3 +34,8 @@ Genetic Algorithm Word Swap
 .. automodule:: textattack.search_methods.genetic_algorithm
    :members:
 
+Particle Swarm Optimization
+##############################
+
+.. automodule:: textattack.search_methods.particle_swarm_optimization
+   :members:

docs/conf.py

@@ -22,7 +22,7 @@
 author = "UVA QData Lab"
 
 # The full version, including alpha/beta/rc tags
-release = "0.1.5"
+release = "0.2.0"
 
 # Set master doc to `index.rst`.
 master_doc = "index"

tests/sample_outputs/interactive_mode.txt

@@ -1,4 +1,4 @@
-Attack(
+/.*/Attack(
   (search_method): GreedyWordSwapWIR(
     (wir_method):  unk
   )
@@ -36,7 +36,7 @@ Attack(
       )
   (is_black_box):  True
 ) 
-
+/.*/
 Running in interactive mode
 ----------------------------
 Enter a sentence to attack or "q" to quit:

tests/sample_outputs/run_attack_deepwordbug_lstm_mr_2.txt

@@ -3,7 +3,20 @@
     (wir_method):  unk
   )
   (goal_function):  UntargetedClassification
-  (transformation):  CompositeTransformation
+  (transformation):  CompositeTransformation(
+    (0): WordSwapNeighboringCharacterSwap(
+        (random_one):  True
+      )
+    (1): WordSwapRandomCharacterSubstitution(
+        (random_one):  True
+      )
+    (2): WordSwapRandomCharacterDeletion(
+        (random_one):  True
+      )
+    (3): WordSwapRandomCharacterInsertion(
+        (random_one):  True
+      )
+    )
   (constraints): 
     (0): LevenshteinEditDistance(
         (max_edit_distance):  30

tests/sample_outputs/run_attack_from_file.txt

@@ -3,7 +3,9 @@
     (wir_method):  unk
   )
   (goal_function):  UntargetedClassification
-  (transformation):  WordSwapRandomCharacterSubstitution
+  (transformation):  WordSwapRandomCharacterSubstitution(
+    (random_one):  True
+  )
   (constraints): None
   (is_black_box):  True
 ) 

tests/sample_outputs/run_attack_transformers_nlp.txt

@@ -3,7 +3,20 @@
     (wir_method):  unk
   )
   (goal_function):  UntargetedClassification
-  (transformation):  CompositeTransformation
+  (transformation):  CompositeTransformation(
+    (0): WordSwapNeighboringCharacterSwap(
+        (random_one):  True
+      )
+    (1): WordSwapRandomCharacterSubstitution(
+        (random_one):  True
+      )
+    (2): WordSwapRandomCharacterDeletion(
+        (random_one):  True
+      )
+    (3): WordSwapRandomCharacterInsertion(
+        (random_one):  True
+      )
+    )
   (constraints): 
     (0): LevenshteinEditDistance(
         (max_edit_distance):  30

textattack/attack_recipes/__init__.py

@@ -12,4 +12,4 @@
 from .pwws_ren_2019 import PWWSRen2019
 from .iga_wang_2019 import IGAWang2019
 from .pruthi_2019 import Pruthi2019
-from .PSO_zang_2020 import PSOZang2020
+from .pso_zang_2020 import PSOZang2020

textattack/attack_recipes/PSO_zang_2020.py → textattack/attack_recipes/pso_zang_2020.py

@@ -4,7 +4,7 @@
     StopwordModification,
 )
 from textattack.goal_functions import UntargetedClassification
-from textattack.search_methods import PSOAlgorithm
+from textattack.search_methods import ParticleSwarmOptimization
 from textattack.shared.attack import Attack
 from textattack.transformations import WordSwapEmbedding, WordSwapHowNet
 
@@ -51,6 +51,6 @@ def PSOZang2020(model):
     #
     # Perform word substitution with a Particle Swarm Optimization (PSO) algorithm.
     #
-    search_method = PSOAlgorithm(pop_size=60, max_iters=20)
+    search_method = ParticleSwarmOptimization(pop_size=60, max_iters=20)
 
     return Attack(goal_function, constraints, transformation, search_method)

textattack/attack_results/attack_result.py

@@ -1,8 +1,10 @@
+from abc import ABC
+
 from textattack.goal_function_results import GoalFunctionResult
 from textattack.shared import utils
 
 
-class AttackResult:
+class AttackResult(ABC):
     """
     Result of an Attack run on a single (output, text_input) pair. 
 

textattack/datasets/dataset.py

@@ -1,21 +1,16 @@
+from abc import ABC, abstractmethod
 import pickle
 import random
 
 from textattack.shared import utils
 
 
-class TextAttackDataset:
+class TextAttackDataset(ABC):
     """
     Any iterable of (label, text_input) pairs qualifies as 
     a ``TextAttackDataset``.
     """
 
-    def __init__(self):
-        """ 
-        Loads a full dataset from disk. 
-        """
-        raise NotImplementedError()
-
     def __iter__(self):
         return self
 

textattack/goal_function_results/goal_function_result.py

@@ -1,3 +1,5 @@
+from abc import ABC, abstractmethod
+
 import torch
 
 
@@ -8,7 +10,7 @@ class GoalFunctionResultStatus:
     SKIPPED = 3
 
 
-class GoalFunctionResult:
+class GoalFunctionResult(ABC):
     """
     Represents the result of a goal function evaluating a AttackedText object.
     
@@ -45,18 +47,21 @@ def __init__(
         if isinstance(self.score, torch.Tensor):
             self.score = self.score.item()
 
+    @abstractmethod
     def get_text_color_input(self):
         """ A string representing the color this result's changed
             portion should be if it represents the original input.
         """
         raise NotImplementedError()
 
+    @abstractmethod
     def get_text_color_perturbed(self):
         """ A string representing the color this result's changed
             portion should be if it represents the perturbed input.
         """
         raise NotImplementedError()
 
+    @abstractmethod
     def get_colored_output(self, color_method=None):
         """ Returns a string representation of this result's output, colored 
             according to `color_method`.

textattack/goal_functions/classification/input_reduction.py

@@ -41,4 +41,7 @@ def _get_score(self, model_output, attacked_text):
         return min(num_words_score + model_score / initial_num_words, 1)
 
     def extra_repr_keys(self):
-        return ["target_num_words"]
+        if self.maximizable:
+            return ["maximizable"]
+        else:
+            return ["maximizable", "target_num_words"]

textattack/goal_functions/classification/targeted_classification.py

@@ -25,4 +25,7 @@ def _get_score(self, model_output, _):
             return model_output[self.target_class]
 
     def extra_repr_keys(self):
-        return ["target_class"]
+        if self.maximizable:
+            return ["maximizable", "target_class"]
+        else:
+            return ["target_class"]

textattack/goal_functions/goal_function.py

@@ -202,8 +202,11 @@ def _call_model(self, attacked_text_list):
             return all_outputs
 
     def extra_repr_keys(self):
+        attrs = []
         if self.query_budget < float("inf"):
-            return ["query_budget"]
-        return []
+            attrs.append("query_budget")
+        if self.maximizable:
+            attrs.append("maximizable")
+        return attrs
 
     __repr__ = __str__ = default_class_repr

textattack/loggers/logger.py

@@ -1,4 +1,7 @@
-class Logger:
+from abc import ABC
+
+
+class Logger(ABC):
     """ An abstract class for different methods of logging attack results. 
     """
 

textattack/search_methods/__init__.py

@@ -3,4 +3,4 @@
 from .greedy_search import GreedySearch
 from .greedy_word_swap_wir import GreedyWordSwapWIR
 from .genetic_algorithm import GeneticAlgorithm
-from .PSO_algorithm import PSOAlgorithm
+from .particle_swarm_optimization import ParticleSwarmOptimization

textattack/search_methods/PSO_algorithm.py → textattack/search_methods/particle_swarm_optimization.py

@@ -13,7 +13,7 @@
 from textattack.search_methods import SearchMethod
 
 
-class PSOAlgorithm(SearchMethod):
+class ParticleSwarmOptimization(SearchMethod):
     """
     Attacks a model with word substiutitions using a Particle Swarm Optimization (PSO) algorithm.
     Some key hyper-parameters are setup according to the original paper:

textattack/transformations/composite_transformation.py

@@ -1,5 +1,6 @@
 import numpy as np
 
+from textattack.shared import utils
 from textattack.transformations.transformation import Transformation
 
 
@@ -34,3 +35,14 @@ def __call__(self, *args, **kwargs):
         for transformation in self.transformations:
             new_attacked_texts.update(transformation(*args, **kwargs))
         return list(new_attacked_texts)
+
+    def __repr__(self):
+        main_str = "CompositeTransformation" + "("
+        transformation_lines = []
+        for i, transformation in enumerate(self.transformations):
+            transformation_lines.append(utils.add_indent(f"({i}): {transformation}", 2))
+        transformation_lines.append(")")
+        main_str += utils.add_indent("\n" + "\n".join(transformation_lines), 2)
+        return main_str
+
+    __str__ = __repr__

textattack/transformations/word_swap_random_character_insertion.py

@@ -48,3 +48,6 @@ def _get_replacement_words(self, word):
                 candidate_words.append(candidate_word)
 
         return candidate_words
+
+    def extra_repr_keys(self):
+        return super().extra_repr_keys() + ["random_one"]

textattack/transformations/word_swap_random_character_substitution.py

@@ -36,3 +36,6 @@ def _get_replacement_words(self, word):
                 candidate_words.append(candidate_word)
 
         return candidate_words
+
+    def extra_repr_keys(self):
+        return super().extra_repr_keys() + ["random_one"]