Revert "Python: switch to shared implementation of IncompleteHostnameRegExp.ql"

This reverts commit ce50f35.

Author: aibaars

config/identical-files.json

@@ -518,7 +518,6 @@
   ],
   "Hostname Regexp queries": [
     "javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
-    "ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll",
-    "python/ql/src/Security/CWE-020/HostnameRegexpShared.qll"
+    "ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll"
   ]
 }

python/ql/lib/semmle/python/RegexTreeView.qll

@@ -2,7 +2,6 @@
 
 import python
 private import semmle.python.regex
-private import semmle.python.dataflow.new.DataFlow
 
 /**
  * An element containing a regular expression term, that is, either
@@ -49,19 +48,6 @@ newtype TRegExpParent =
   /** A back reference */
   TRegExpBackRef(Regex re, int start, int end) { re.backreference(start, end) }
 
-/**
- * Provides utility predicates related to regular expressions.
- */
-module RegExpPatterns {
-  /**
-   * Gets a pattern that matches common top-level domain names in lower case.
-   */
-  string getACommonTld() {
-    // according to ranking by http://google.com/search?q=site:.<<TLD>>
-    result = "(?:com|org|edu|gov|uk|net|io)(?![a-z0-9])"
-  }
-}
-
 /**
  * An element containing a regular expression term, that is, either
  * a string literal (parsed as a regular expression)
@@ -459,8 +445,6 @@ class RegExpAlt extends RegExpTerm, TRegExpAlt {
   override string getPrimaryQLClass() { result = "RegExpAlt" }
 }
 
-class RegExpCharEscape = RegExpEscape;
-
 /**
  * An escaped regular expression term, that is, a regular expression
  * term starting with a backslash, which is not a backreference.
@@ -767,9 +751,6 @@ class RegExpGroup extends RegExpTerm, TRegExpGroup {
    */
   int getNumber() { result = re.getGroupNumber(start, end) }
 
-  /** Holds if this is a capture group. */
-  predicate isCapture() { exists(this.getNumber()) }
-
   /** Holds if this is a named capture group. */
   predicate isNamed() { exists(this.getName()) }
 
@@ -1028,24 +1009,3 @@ class RegExpBackRef extends RegExpTerm, TRegExpBackRef {
 
 /** Gets the parse tree resulting from parsing `re`, if such has been constructed. */
 RegExpTerm getParsedRegExp(StrConst re) { result.getRegex() = re and result.isRootTerm() }
-
-/**
- * A node whose value may flow to a position where it is interpreted
- * as a part of a regular expression.
- */
-class RegExpPatternSource extends DataFlow::CfgNode {
-  private Regex astNode;
-
-  RegExpPatternSource() { astNode = this.asExpr() }
-
-  /**
-   * Gets a node where the pattern of this node is parsed as a part of
-   * a regular expression.
-   */
-  DataFlow::Node getAParse() { result = this }
-
-  /**
-   * Gets the root term of the regular expression parsed from this pattern.
-   */
-  RegExpTerm getRegExpTerm() { result.getRegex() = astNode }
-}

python/ql/src/Security/CWE-020/HostnameRegexpShared.qll

@@ -8,9 +8,35 @@
  * @id py/incomplete-hostname-regexp
  * @tags correctness
  *       security
- *       external/cwe/cwe-020
+ *       external/cwe/cwe-20
  */
 
-import HostnameRegexpShared
+import python
+import semmle.python.regex
 
-query predicate problems = incompleteHostnameRegExp/4;
+private string commonTopLevelDomainRegex() { result = "com|org|edu|gov|uk|net|io" }
+
+/**
+ * Holds if `pattern` is a regular expression pattern for URLs with a host matched by `hostPart`,
+ * and `pattern` contains a subtle mistake that allows it to match unexpected hosts.
+ */
+bindingset[pattern]
+predicate isIncompleteHostNameRegExpPattern(string pattern, string hostPart) {
+  hostPart =
+    pattern
+        .regexpCapture("(?i).*" +
+            // an unescaped single `.`
+            "(?<!\\\\)[.]" +
+            // immediately followed by a sequence of subdomains, perhaps with some regex characters mixed in, followed by a known TLD
+            "([():|?a-z0-9-]+(\\\\)?[.](" + commonTopLevelDomainRegex() + "))" + ".*", 1)
+}
+
+from Regex r, string pattern, string hostPart
+where
+  r.getText() = pattern and
+  isIncompleteHostNameRegExpPattern(pattern, hostPart) and
+  // ignore patterns with capture groups after the TLD
+  not pattern.regexpMatch("(?i).*[.](" + commonTopLevelDomainRegex() + ").*[(][?]:.*[)].*")
+select r,
+  "This regular expression has an unescaped '.' before '" + hostPart +
+    "', so it might match more hosts than expected."

python/ql/src/Security/CWE-020/HostnameRegexpSpecific.qll

@@ -1 +1 @@
-| hosttest.py:6:31:6:53 | (www\|beta).example.com/ | This regular expression has an unescaped '.' before 'example.com/', so it might match more hosts than expected. | hosttest.py:6:27:6:51 | ControlFlowNode for Str | here |
+| hosttest.py:6:27:6:51 | Str | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. |