wip: tests pass

coverage/parser.py

@@ -25,7 +25,7 @@
 from coverage.bytecode import code_objects
 from coverage.debug import short_stack
 from coverage.exceptions import NoSource, NotPython
-from coverage.misc import join_regex, nice_pair
+from coverage.misc import nice_pair
 from coverage.phystokens import generate_tokens
 from coverage.types import TArc, TLineNo
 
@@ -124,34 +124,18 @@ def _raw_parse(self) -> None:
  # Find lines which match an exclusion pattern.
  if self.exclude:
  self.raw_excluded = self.lines_matching(self.exclude)
-
- # Tokenize, to find excluded suites, to find docstrings, and to find
- # multi-line statements.
- # Attributes updated:
- # self.raw_classdefs
- # lines with `class` definitions
- # self.raw_excluded
- # lines with pragmas, and others due to being part of decorators
- # or bodies of excluded colon-lines
- # self.raw_docstrings
- # lines of docstrings
- # self._multiline
- # self.raw_statements
+ self.excluded = set(self.raw_excluded)
 
  # The current number of indents.
  indent: int = 0
  # An exclusion comment will exclude an entire clause at this indent.
  exclude_indent: int = 0
  # Are we currently excluding lines?
  excluding: bool = False
- # Are we excluding decorators now?
- excluding_decorators: bool = False
  # The line number of the first line in a multi-line statement.
  first_line: int = 0
  # Is the file empty?
  empty: bool = True
- # Is this the first token on a line?
- first_on_line: bool = True
  # Parenthesis (and bracket) nesting level.
  nesting: int = 0
 
@@ -170,23 +154,15 @@ def _raw_parse(self) -> None:
  elif toktype == token.OP:
  if ttext == ":" and nesting == 0:
  should_exclude = (
- self.raw_excluded.intersection(range(first_line, elineno + 1))
- or excluding_decorators
+ self.excluded.intersection(range(first_line, elineno + 1))
  )
  if not excluding and should_exclude:
  # Start excluding a suite. We trigger off of the colon
  # token so that the #pragma comment will be recognized on
  # the same line as the colon.
- self.raw_excluded.add(elineno)
+ self.excluded.add(elineno)
  exclude_indent = indent
  excluding = True
- excluding_decorators = False
- elif ttext == "@" and first_on_line:
- # A decorator.
- if elineno in self.raw_excluded:
- excluding_decorators = True
- if excluding_decorators:
- self.raw_excluded.add(elineno)
  elif ttext in "([{":
  nesting += 1
  elif ttext in ")]}":
@@ -199,7 +175,6 @@ def _raw_parse(self) -> None:
  for l in range(first_line, elineno+1):
  self._multiline[l] = first_line
  first_line = 0
- first_on_line = True
 
  if ttext.strip() and toktype != tokenize.COMMENT:
  # A non-white-space token.
@@ -211,8 +186,7 @@ def _raw_parse(self) -> None:
  if excluding and indent <= exclude_indent:
  excluding = False
  if excluding:
- self.raw_excluded.add(elineno)
- first_on_line = False
+ self.excluded.add(elineno)
 
  # Find the starts of the executable statements.
  if not empty:
@@ -225,15 +199,15 @@ def _raw_parse(self) -> None:
  if env.PYBEHAVIOR.module_firstline_1 and self._multiline:
  self._multiline[1] = min(self.raw_statements)
 
- # Hacked-in ast version
+ self.excluded = self.first_lines(self.excluded)
 
- classdefs2 = set()
- docstrings2 = set()
- multiline = {}
- root = ast.parse(self.text)
- for node in ast.walk(root):
+ # AST lets us find classes, docstrings, and decorator-affected
+ # functions and classes.
+ for node in ast.walk(self._ast_root):
+ # Find class definitions.
  if isinstance(node, ast.ClassDef):
- classdefs2.add(node.lineno)
+ self.raw_classdefs.add(node.lineno)
+ # Find docstrings.
  if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef, ast.Module)):
  if node.body:
  first = node.body[0]
@@ -242,12 +216,15 @@ def _raw_parse(self) -> None:
  and isinstance(first.value, ast.Constant)
  and isinstance(first.value.value, str)
  ):
- docstrings2.update(range(first.lineno, first.end_lineno+1))
- #assert classdefs2 == self.raw_classdefs
- #assert docstrings2 == self.raw_docstrings, f"{self.raw_docstrings = }, {docstrings2 = } in {self.filename}"
- self.raw_classdefs = classdefs2
- self.raw_docstrings = docstrings2
-
+ self.raw_docstrings.update(
+ range(first.lineno, cast(int, first.end_lineno) + 1)
+ )
+ # Exclusions carry from decorators and signatures to the bodies of
+ # functions and classes.
+ if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+ first_line = min((d.lineno for d in node.decorator_list), default=node.lineno)
+ if self.excluded.intersection(range(first_line, node.lineno + 1)):
+ self.excluded.update(range(first_line, cast(int, node.end_lineno) + 1))
 
  @functools.lru_cache(maxsize=1000)
  def first_line(self, lineno: TLineNo) -> TLineNo:
@@ -283,6 +260,7 @@ def parse_source(self) -> None:
 
  """
  try:
+ self._ast_root = ast.parse(self.text)
  self._raw_parse()
  except (tokenize.TokenError, IndentationError, SyntaxError) as err:
  if hasattr(err, "lineno"):
@@ -294,8 +272,6 @@ def parse_source(self) -> None:
  f"{err.args[0]!r} at line {lineno}",
  ) from err
 
- self.excluded = self.first_lines(self.raw_excluded)
-
  ignore = self.excluded | self.raw_docstrings
  starts = self.raw_statements - ignore
  self.statements = self.first_lines(starts) - ignore
@@ -318,7 +294,7 @@ def _analyze_ast(self) -> None:
  `_all_arcs` is the set of arcs in the code.
 
  """
- aaa = AstArcAnalyzer(self.text, self.raw_statements, self._multiline)
+ aaa = AstArcAnalyzer(self._ast_root, self.raw_statements, self._multiline)
  aaa.analyze()
 
  self._all_arcs = set()
@@ -700,11 +676,11 @@ class AstArcAnalyzer:
 
  def __init__(
  self,
- text: str,
+ root_node: ast.AST,
  statements: set[TLineNo],
  multiline: dict[TLineNo, TLineNo],
  ) -> None:
- self.root_node = ast.parse(text)
+ self.root_node = root_node
  # TODO: I think this is happening in too many places.
  self.statements = {multiline.get(l, l) for l in statements}
  self.multiline = multiline

lab/parser.py

@@ -96,7 +96,7 @@ def one_file(self, options, filename):
  exit_counts = pyparser.exit_counts()
 
  for lineno, ltext in enumerate(pyparser.lines, start=1):
- marks = [' ', ' ', ' ', ' ', ' ', ' ']
+ marks = [' '] * 6
  a = ' '
  if lineno in pyparser.raw_statements:
  marks[0] = '-'
@@ -110,7 +110,9 @@ def one_file(self, options, filename):
  if lineno in pyparser.raw_classdefs:
  marks[3] = 'C'
  if lineno in pyparser.raw_excluded:
- marks[4] = 'x'
+ marks[4] = 'X'
+ elif lineno in pyparser.excluded:
+ marks[4] = '×'
  if lineno in pyparser._multiline.values():
  marks[5] = 'o'
  elif lineno in pyparser._multiline.keys():

tests/test_parser.py

@@ -125,10 +125,7 @@ def foo():
  assert parser.exit_counts() == { 1:1, 2:1, 3:1, 6:1 }
 
  def test_indentation_error(self) -> None:
- msg = (
- "Couldn't parse '<code>' as Python source: " +
- "'unindent does not match any outer indentation level.*' at line 3"
- )
+ msg = r"Couldn't parse '<code>' as Python source: '.*' at line \d+"
  with pytest.raises(NotPython, match=msg):
  _ = self.parse_text("""\
  0 spaces
@@ -137,15 +134,7 @@ def test_indentation_error(self) -> None:
  """)
 
  def test_token_error(self) -> None:
- submsgs = [
- r"EOF in multi-line string", # before 3.12.0b1
- r"unterminated triple-quoted string literal .detected at line 1.", # after 3.12.0b1
- ]
- msg = (
- r"Couldn't parse '<code>' as Python source: '"
- + r"(" + "|".join(submsgs) + ")"
- + r"' at line 1"
- )
+ msg = r"Couldn't parse '<code>' as Python source: '.*' at line \d+"
  with pytest.raises(NotPython, match=msg):
  _ = self.parse_text("'''")
 
@@ -754,6 +743,22 @@ def __init__(self):
  assert parser.raw_statements == {1, 2, 3, 5, 6, 7, 8}
  assert parser.statements == {1, 2, 3}
 
+ def test_over_exclusion_bug1779(self) -> None:
+ # https://github.com/nedbat/coveragepy/issues/1779
+ parser = self.parse_text("""\
+ import abc
+
+ class MyProtocol: # nocover 3
+ @abc.abstractmethod # nocover 4
+ def my_method(self) -> int:
+ ... # 6
+
+ def function() -> int:
+ return 9
+ """)
+ assert parser.raw_statements == {1, 3, 4, 5, 6, 8, 9}
+ assert parser.statements == {1, 8, 9}
+
 
 class ParserMissingArcDescriptionTest(PythonParserTestBase):
  """Tests for PythonParser.missing_arc_description."""