Merge pull request #66 from haveibeensquatted/64-infinite-permutation…

…-generation-on-three-dot-domains

fix(permutation): remove aggressive idna filtering

twistrs/Cargo.toml

@@ -28,8 +28,6 @@ phf = { version = "0.11.2", features = ["macros"] }
 async-smtp = { version = "0.9.0", optional = true }
 futures = "0.3.28"
 tokio = { version = "1.29.1", features = ["full"] }
-fancy-regex = "0.11.0"
-idna = "0.4.0"
 hyper = { version = "0.14.20", features = ["client", "http1", "http2", "tcp"] }
 itertools = "0.11.0"
 serde = { version = "1.0.171", default_features = false, features = ["derive"]}

twistrs/src/constants.rs

@@ -1,4 +1,3 @@
-use fancy_regex::Regex;
 use phf::phf_map;
 
 #[cfg(feature = "whois_lookup")]
@@ -12,13 +11,6 @@ use hyper::client::HttpConnector;
 include!(concat!(env!("OUT_DIR"), "/data.rs"));
 
 lazy_static! {
-
- /// IDNA filter regex used to reduce number of domain permutations
- /// that are generated and validated.
- ///
- /// The regex is taken from [dnstwist](https://github.com/elceef/dnstwist/blob/5368e465c35355c43d189b093acf41773e869d25/dnstwist.py#L213-L227).
- pub static ref IDNA_FILTER_REGEX: Regex = Regex::new("(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{1,63}(?<!-)\\.)+[a-zA-Z]{2,63}\\.?$)").unwrap();
-
  pub static ref KEYBOARD_LAYOUTS: Vec<&'static phf::Map<char, &'static str>> = vec![
  &QWERTY_KEYBOARD_LAYOUT,
  &QWERTZ_KEYBOARD_LAYOUT,

twistrs/src/enrich.rs

@@ -303,7 +303,7 @@ impl DomainMetadata {
 
  match &self.ips {
  Some(ips) => {
- for ip in ips.iter() {
+ for ip in ips {
  if let Ok(lookup_result) = geoip.lookup::<geoip2::City>(*ip) {
  let mut geoip_string = String::new();