Patch indentation when removing braces

If the first indentation of the region is greater than the indentation
of the enclosing region, we use it to indent the whole region.
Otherwise we use the incremented indentation of the enclosing region.

```scala
  def foo = {
        x // we replicate indentation of x downward in region
    y
  }
```

```scala
  def foo = {
x // indentation of x is incorrect, we increment enclosing indentation
  y
  }
```

A bigger indentation than the required one is permitted except
just after a closing brace.

```scala
def bar = {
  x
    .toString // permitted indentation
  def foo = {
  }
    bar //  must be unindented, to not fall into the body of foo
}
```

And other bug fixes (see scala#17522)

Author: adpi2

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -60,15 +60,17 @@ object Scanners {
     /** the base of a number */
     var base: Int = 0
 
-    def copyFrom(td: TokenData): Unit = {
+    def copyFrom(td: TokenData): this.type =
       this.token = td.token
       this.offset = td.offset
       this.lastOffset = td.lastOffset
       this.lineOffset = td.lineOffset
       this.name = td.name
       this.strVal = td.strVal
       this.base = td.base
-    }
+      this
+
+    def saveCopy: TokenData = newTokenData.copyFrom(this)
 
     def isNewLine = token == NEWLINE || token == NEWLINES
     def isStatSep = isNewLine || token == SEMI
@@ -86,12 +88,14 @@ object Scanners {
 
     def isOperator =
       token == BACKQUOTED_IDENT
-      || token == IDENTIFIER && isOperatorPart(name(name.length - 1))
+      || token == IDENTIFIER && isOperatorPart(name.last)
 
     def isArrow =
       token == ARROW || token == CTXARROW
   }
 
+  def newTokenData: TokenData = new TokenData {}
+
   abstract class ScannerCommon(source: SourceFile)(using Context) extends CharArrayReader with TokenData {
     val buf: Array[Char] = source.content
     def nextToken(): Unit
@@ -264,11 +268,10 @@ object Scanners {
       if (idx >= 0 && idx <= lastKeywordStart) handleMigration(kwArray(idx))
       else IDENTIFIER
 
-    def newTokenData: TokenData = new TokenData {}
-
     /** We need one token lookahead and one token history
      */
     val next = newTokenData
+    val last = newTokenData
     private val prev = newTokenData
 
     /** The current region. This is initially an Indented region with zero indentation width. */
@@ -385,12 +388,11 @@ object Scanners {
     /** Produce next token, filling TokenData fields of Scanner.
      */
     def nextToken(): Unit =
-      val lastToken = token
-      val lastName = name
-      adjustSepRegions(lastToken)
-      getNextToken(lastToken)
-      if isAfterLineEnd then handleNewLine(lastToken)
-      postProcessToken(lastToken, lastName)
+      last.copyFrom(this)
+      adjustSepRegions(last.token)
+      getNextToken(last.token)
+      if isAfterLineEnd then handleNewLine()
+      postProcessToken()
       profile.recordNewToken()
       printState()
 
@@ -433,7 +435,7 @@ object Scanners {
         // in backticks and is a binary operator. Hence, `x` is not classified as a
         // leading infix operator.
         def assumeStartsExpr(lexeme: TokenData) =
-          (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONeol)
+          (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONfollow)
           && (!lexeme.isOperator || nme.raw.isUnary(lexeme.name))
         val lookahead = LookaheadScanner()
         lookahead.allowLeadingInfixOperators = false
@@ -483,7 +485,7 @@ object Scanners {
             if (nextChar == ch)
               recur(idx - 1, ch, n + 1, k)
             else {
-              val k1: IndentWidth => IndentWidth = if (n == 0) k else Conc(_, Run(ch, n))
+              val k1: IndentWidth => IndentWidth = if (n == 0) k else iw => k(Conc(iw, Run(ch, n)))
               recur(idx - 1, nextChar, 1, k1)
             }
           else recur(idx - 1, ' ', 0, identity)
@@ -523,7 +525,7 @@ object Scanners {
      *
      *      The following tokens can start an indentation region:
      *
-     *         :  =  =>  <-  if  then  else  while  do  try  catch  
+     *         :  =  =>  <-  if  then  else  while  do  try  catch
      *         finally  for  yield  match  throw  return  with
      *
      *      Inserting an INDENT starts a new indentation region with the indentation of the current
@@ -547,7 +549,7 @@ object Scanners {
      *  I.e. `a <= b` iff `b.startsWith(a)`. If indentation is significant it is considered an error
      *  if the current indentation width and the indentation of the current token are incomparable.
      */
-    def handleNewLine(lastToken: Token) =
+    def handleNewLine() =
       var indentIsSignificant = false
       var newlineIsSeparating = false
       var lastWidth = IndentWidth.Zero
@@ -576,7 +578,7 @@ object Scanners {
        */
       inline def isContinuing =
         lastWidth < nextWidth
-        && (openParensTokens.contains(token) || lastToken == RETURN)
+        && (openParensTokens.contains(token) || last.token == RETURN)
         && !pastBlankLine
         && !migrateTo3
         && !noindentSyntax
@@ -590,12 +592,12 @@ object Scanners {
         case _: InString => ()
         case r =>
           indentIsSignificant = indentSyntax
-          r.proposeKnownWidth(nextWidth, lastToken)
+          r.proposeKnownWidth(nextWidth, last.token)
           lastWidth = r.knownWidth
           newlineIsSeparating = r.isInstanceOf[InBraces]
 
       if newlineIsSeparating
-         && canEndStatTokens.contains(lastToken)
+         && canEndStatTokens.contains(last.token)
          && canStartStatTokens.contains(token)
          && !isLeadingInfixOperator(nextWidth)
          && !isContinuing
@@ -606,7 +608,7 @@ object Scanners {
            || nextWidth == lastWidth && (indentPrefix == MATCH || indentPrefix == CATCH) && token != CASE then
           if currentRegion.isOutermost then
             if nextWidth < lastWidth then currentRegion = topLevelRegion(nextWidth)
-          else if !isLeadingInfixOperator(nextWidth) && !statCtdTokens.contains(lastToken) && lastToken != INDENT then
+          else if !isLeadingInfixOperator(nextWidth) && !statCtdTokens.contains(last.token) && last.token != INDENT then
             currentRegion match
               case r: Indented =>
                 insert(OUTDENT, offset)
@@ -630,15 +632,16 @@ object Scanners {
                   report.warning("Line is indented too far to the left, or a `}` is missing", sourcePos())
 
         else if lastWidth < nextWidth
-             || lastWidth == nextWidth && (lastToken == MATCH || lastToken == CATCH) && token == CASE then
-          if canStartIndentTokens.contains(lastToken) then
-            currentRegion = Indented(nextWidth, lastToken, currentRegion)
+             || lastWidth == nextWidth && (last.token == MATCH || last.token == CATCH) && token == CASE then
+          if canStartIndentTokens.contains(last.token) then
+            currentRegion = Indented(nextWidth, last.token, currentRegion)
             insert(INDENT, offset)
-          else if lastToken == SELFARROW then
+          else if last.token == SELFARROW then
             currentRegion.knownWidth = nextWidth
         else if (lastWidth != nextWidth)
           val lw = lastWidth
-          errorButContinue(spaceTabMismatchMsg(lw, nextWidth))
+          val msg = spaceTabMismatchMsg(lw, nextWidth)
+          if rewriteToIndent then report.warning(msg) else errorButContinue(msg)
       if token != OUTDENT then
         handleNewIndentWidth(currentRegion, _.otherIndentWidths += nextWidth)
       if next.token == EMPTY then
@@ -702,7 +705,7 @@ object Scanners {
      *         SEMI + ELSE => ELSE, COLON following id/)/] => COLONfollow
      *  - Insert missing OUTDENTs at EOF
      */
-    def postProcessToken(lastToken: Token, lastName: SimpleName): Unit = {
+    def postProcessToken(): Unit = {
       def fuse(tok: Int) = {
         token = tok
         offset = prev.offset
@@ -738,8 +741,8 @@ object Scanners {
         case END =>
           if !isEndMarker then token = IDENTIFIER
         case COLONop =>
-          if lastToken == IDENTIFIER && lastName != null && isIdentifierStart(lastName.head)
-              || colonEOLPredecessors.contains(lastToken)
+          if last.token == IDENTIFIER && last.name != null && isIdentifierStart(last.name.head)
+              || colonEOLPredecessors.contains(last.token)
           then token = COLONfollow
         case RBRACE | RPAREN | RBRACKET =>
           closeIndented()
@@ -758,6 +761,8 @@ object Scanners {
         if endMarkerTokens.contains(lookahead.token)
           && source.offsetToLine(lookahead.offset) == endLine
         then
+          if rewriteToIndent && lookahead.token == MATCH then
+            patch(Span(offset, offset + 3), "`end`")
           lookahead.nextToken()
           if lookahead.token == EOF
           || source.offsetToLine(lookahead.offset) > endLine
@@ -1266,6 +1271,7 @@ object Scanners {
               putChar(ch) ; nextRawChar()
               loopRest()
             else
+              next.lineOffset = if next.lastOffset < lineStartOffset then lineStartOffset else -1
               finishNamedToken(IDENTIFIER, target = next)
           end loopRest
           setStrVal()
@@ -1312,10 +1318,10 @@ object Scanners {
       }
     end getStringPart
 
-    private def fetchStringPart(multiLine: Boolean) = {
+    private def fetchStringPart(multiLine: Boolean) =
       offset = charOffset - 1
+      lineOffset = if lastOffset < lineStartOffset then lineStartOffset else -1
       getStringPart(multiLine)
-    }
 
     private def isTripleQuote(): Boolean =
       if (ch == '"') {
@@ -1657,21 +1663,31 @@ object Scanners {
     case Run(ch: Char, n: Int)
     case Conc(l: IndentWidth, r: Run)
 
-    def <= (that: IndentWidth): Boolean = this match {
-      case Run(ch1, n1) =>
-        that match {
-          case Run(ch2, n2) => n1 <= n2 && (ch1 == ch2 || n1 == 0)
-          case Conc(l, r) => this <= l
-        }
-      case Conc(l1, r1) =>
-        that match {
-          case Conc(l2, r2) => l1 == l2 && r1 <= r2
-          case _ => false
-        }
-    }
+    def <= (that: IndentWidth): Boolean = (this, that) match
+      case (Run(ch1, n1), Run(ch2, n2)) => n1 <= n2 && (ch1 == ch2 || n1 == 0)
+      case (Conc(l1, r1), Conc(l2, r2)) => (l1 == l2 && r1 <= r2) || this <= l2
+      case (_, Conc(l2, _)) => this <= l2
+      case _ => false
 
     def < (that: IndentWidth): Boolean = this <= that && !(that <= this)
 
+    def >= (that: IndentWidth): Boolean = that <= this
+
+    def >(that: IndentWidth): Boolean = that < this
+
+    def size: Int = this match
+      case Run(_, n) => n
+      case Conc(l, r) => l.size + r.n
+
+    /** Add one level of indentation (one tab or two spaces depending on the last char) */
+    def increment: IndentWidth =
+      def incRun(ch: Char, n: Int): Run = ch match
+        case ' ' => IndentWidth.Run(' ', n + 2)
+        case ch => IndentWidth.Run(ch, n + 1)
+      this match
+        case Run(ch, n) => incRun(ch, n)
+        case Conc(l, Run(ch, n)) => Conc(l, incRun(ch, n))
+
     /** Does `this` differ from `that` by not more than a single space? */
     def isClose(that: IndentWidth): Boolean = this match
       case Run(ch1, n1) =>

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -324,8 +324,7 @@ object MarkupParsers {
     /** Some try/catch/finally logic used by xLiteral and xLiteralPattern.  */
     inline private def xLiteralCommon(f: () => Tree, ifTruncated: String => Unit): Tree = {
       assert(parser.in.token == Tokens.XMLSTART)
-      val saved = parser.in.newTokenData
-      saved.copyFrom(parser.in)
+      val saved = parser.in.saveCopy
       var output: Tree = null.asInstanceOf[Tree]
       try output = f()
       catch {
@@ -404,7 +403,7 @@ object MarkupParsers {
     def escapeToScala[A](op: => A, kind: String): A = {
       xEmbeddedBlock = false
       val res = saving(parser.in.currentRegion, parser.in.currentRegion = _) {
-        val lbrace = parser.in.newTokenData
+        val lbrace = Scanners.newTokenData
         lbrace.token = LBRACE
         lbrace.offset = parser.in.charOffset - 1
         lbrace.lastOffset = parser.in.lastOffset

compiler/src/dotty/tools/dotc/parsing/xml/MarkupParsers.scala

@@ -25,9 +25,13 @@ object Rewrites {
     def addPatch(span: Span, replacement: String): Unit =
       pbuf += Patch(span, replacement)
 
+    def patchOver(span: Span, replacement: String): Unit =
+      pbuf.indices.reverse.find(i => span.contains(pbuf(i).span)).foreach(pbuf.remove)
+      pbuf += Patch(span, replacement)
+
     def apply(cs: Array[Char]): Array[Char] = {
       val delta = pbuf.map(_.delta).sum
-      val patches = pbuf.toList.sortBy(_.span.start)
+      val patches = pbuf.toList.sortBy(p => (p.span.start, p.span.end))
       if (patches.nonEmpty)
         patches.reduceLeft {(p1, p2) =>
           assert(p1.span.end <= p2.span.start, s"overlapping patches in $source: $p1 and $p2")
@@ -71,6 +75,14 @@ object Rewrites {
          .addPatch(span, replacement)
     )
 
+  /** Record a patch that replaces the first patch that it contains */
+  def patchOver(source: SourceFile, span: Span, replacement: String)(using Context): Unit =
+    if ctx.reporter != Reporter.NoReporter // NoReporter is used for syntax highlighting
+    then ctx.settings.rewrite.value.foreach(_.patched
+         .getOrElseUpdate(source, new Patches(source))
+         .patchOver(span, replacement)
+    )
+
   /** Patch position in `ctx.compilationUnit.source`. */
   def patch(span: Span, replacement: String)(using Context): Unit =
     patch(ctx.compilationUnit.source, span, replacement)

compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala

@@ -82,9 +82,14 @@ class CompilationTests {
       compileFile("tests/rewrites/rewrites3x.scala", defaultOptions.and("-rewrite", "-source", "future-migration")),
       compileFile("tests/rewrites/filtering-fors.scala", defaultOptions.and("-rewrite", "-source", "3.2-migration")),
       compileFile("tests/rewrites/refutable-pattern-bindings.scala", defaultOptions.and("-rewrite", "-source", "3.2-migration")),
-      compileFile("tests/rewrites/i8982.scala", defaultOptions.and("-indent", "-rewrite")),
-      compileFile("tests/rewrites/i9632.scala", defaultOptions.and("-indent", "-rewrite")),
-      compileFile("tests/rewrites/i11895.scala", defaultOptions.and("-indent", "-rewrite")),
+      compileFile("tests/rewrites/i8982.scala", indentRewrite),
+      compileFile("tests/rewrites/i9632.scala", indentRewrite),
+      compileFile("tests/rewrites/i11895.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-rewrite.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-comments.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-mix-tab-space.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-3-spaces.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-mix-brace.scala", indentRewrite),
       compileFile("tests/rewrites/i12340.scala", unindentOptions.and("-rewrite")),
       compileFile("tests/rewrites/i17187.scala", unindentOptions.and("-rewrite")),
       compileFile("tests/rewrites/i17399.scala", unindentOptions.and("-rewrite")),

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -65,6 +65,7 @@ object TestConfiguration {
 
   val commonOptions = Array("-indent") ++ checkOptions ++ noCheckOptions ++ yCheckOptions
   val defaultOptions = TestFlags(basicClasspath, commonOptions)
+  val indentRewrite = defaultOptions.and("-rewrite")
   val unindentOptions = TestFlags(basicClasspath, Array("-no-indent") ++ checkOptions ++ noCheckOptions ++ yCheckOptions)
   val withCompilerOptions =
     defaultOptions.withClasspath(withCompilerClasspath).withRunClasspath(withCompilerClasspath)

compiler/test/dotty/tools/vulpix/TestConfiguration.scala

@@ -152,6 +152,11 @@ object Test23:
   val _ = 1 `+`:  // ok
     x
 
+  // leading infix op
+  val _ = 1
+  `+` :
+    x
+
   val r = 1 to:
     100
 

tests/pos/indent-colons.scala

@@ -0,0 +1,21 @@
+// Rewrite to indent, keeping 3 spaces as indentation
+
+def m1 =
+   def m2 =
+      "" +
+         "" +
+         ""
+   m2
+
+def m4 =
+   def m5 =
+      def m6 =
+         val x = ""
+         x
+            .apply(0)
+            .toString
+      m6
+         .toString
+   m5 +
+      m5
+         .toString

tests/rewrites/indent-3-spaces.check

@@ -0,0 +1,26 @@
+// Rewrite to indent, keeping 3 spaces as indentation
+
+def m1 = {
+   def m2 = {
+      "" +
+         "" +
+         ""
+   }
+   m2
+}
+
+def m4 = {
+   def m5 = {
+      def m6 = {
+         val x = ""
+         x
+            .apply(0)
+            .toString
+      }
+      m6
+         .toString
+   }
+   m5 +
+      m5
+         .toString
+}