diff --git a/default-recommendations/comment-preservation-test.rkt b/default-recommendations/comment-preservation-test.rkt new file mode 100644 index 00000000..c692eb15 --- /dev/null +++ b/default-recommendations/comment-preservation-test.rkt @@ -0,0 +1,104 @@ +#lang resyntax/test + + +require: resyntax/default-recommendations let-replacement + + +header: +- #lang racket/base + + +test: "expression comments are preserved between let bindings" +------------------------------ +(define (foo) + (let ([x 1] + #;(debug-binding [z 0]) + [y 2]) + (+ x y))) +============================== +(define (foo) + (define x 1) + #;(debug-binding [z 0]) + (define y 2) + (+ x y)) +------------------------------ + + +test: "expression comments with atoms are preserved between let bindings" +------------------------------ +(define (foo) + (let ([x 1] + #;unused-binding + [y 2]) + (+ x y))) +============================== +(define (foo) + (define x 1) + #;unused-binding + (define y 2) + (+ x y)) +------------------------------ + + +test: "block comments are preserved between let bindings" +------------------------------ +(define (foo) + (let ([x 1] + #| The second + binding |# + [y 2]) + (+ x y))) +============================== +(define (foo) + (define x 1) + #| The second + binding |# + (define y 2) + (+ x y)) +------------------------------ + + +test: "expression comments with nested sexps are preserved" +------------------------------ +(define (foo) + (let ([x 1] + #;(commented-binding + [y 3] + [z 4]) + [a 2]) + (+ x a))) +============================== +(define (foo) + (define x 1) + #;(commented-binding [y 3] [z 4]) + (define a 2) + (+ x a)) +------------------------------ + + +test: "expression comments in let body are preserved" +------------------------------ +(define (foo) + (let ([x 1]) + #;(debug-stmt) + (+ x 1))) +============================== +(define (foo) + (define x 1) + #;(debug-stmt) + (+ x 1)) +------------------------------ + + +test: "block comments in let body are preserved" +------------------------------ +(define (foo) + (let ([x 1]) + #| computation |# + (+ x 1))) +============================== +(define (foo) + (define x 1) + #| computation |# + (+ x 1)) +------------------------------ diff --git a/main.rkt b/main.rkt index bcf381b1..a9db4ebb 100644 --- a/main.rkt +++ b/main.rkt @@ -54,7 +54,6 @@ resyntax/base resyntax/default-recommendations resyntax/private/analysis - resyntax/private/comment-reader resyntax/private/git resyntax/private/limiting resyntax/private/line-replacement @@ -170,7 +169,7 @@ #:suite [suite default-recommendations] #:lines [lines (range-set (unbounded-range #:comparator natural<=>))] #:timeout-ms [timeout-ms 10000]) - (define comments (with-input-from-source source read-comment-locations)) + (define comments (source-comment-locations source)) (define source-lang (source-read-language source)) (guard source-lang #:else (log-resyntax-warning "skipping ~a because its #lang could not be determined" @@ -235,7 +234,7 @@ (define/guard (reysntax-analyze-for-properties-only source #:suite [suite default-recommendations] #:timeout-ms [timeout-ms 10000]) - (define comments (with-input-from-source source read-comment-locations)) + (define comments (source-comment-locations source)) (define full-source (source->string source)) (guard (string-prefix? full-source "#lang racket") #:else (log-resyntax-warning "skipping ~a because it does not start with #lang racket" diff --git a/private/comment-reader.rkt b/private/comment-reader.rkt deleted file mode 100644 index 49b7f95a..00000000 --- a/private/comment-reader.rkt +++ /dev/null @@ -1,129 +0,0 @@ -#lang racket/base - - -(require racket/contract/base) - - -(provide - (contract-out - [read-comment-locations (->* () (input-port?) range-set?)])) - - -(require br-parser-tools/lex - racket/sequence - rebellion/base/comparator - rebellion/base/range - rebellion/collection/list - rebellion/collection/range-set - rebellion/streaming/reducer - rebellion/streaming/transducer - resyntax/private/syntax-traversal - (prefix-in : br-parser-tools/lex-sre)) - - -(module+ test - (require rackunit - (submod ".."))) - - -;@---------------------------------------------------------------------------------------------------- - - -(define (read-comment-locations [in (current-input-port)]) - (port-count-lines! in) - (define (next!) - (comment-lexer in)) - (transduce (in-producer next! eof) - (mapping srcloc-token-srcloc) - (mapping srcloc-range) - #:into (into-range-set natural<=>))) - - -(define (srcloc-range srcloc) - (define start (sub1 (srcloc-position srcloc))) - (define end (+ start (srcloc-span srcloc))) - (closed-open-range start end #:comparator natural<=>)) - - -(define-tokens racket-tokens (LINE-COMMENT BLOCK-COMMENT)) - - -(define-lex-abbrev racket-line-comment - (concatenation ";" (complement (:: any-string "\n" any-string)) "\n")) - - -(define (build-racket-line-comment lexeme) - (token-LINE-COMMENT (string->immutable-string lexeme))) - - -;; Technically not correct because block comments can be nested. -(define-lex-abbrev racket-block-comment - (concatenation "#|" (complement (:: any-string (:or "#|" "#|") any-string)) "|#")) - - -(define (build-racket-block-comment lexeme) - (token-BLOCK-COMMENT (string->immutable-string lexeme))) - - -;; This lexer should also read string literals and discard them, so that comment-starting characters -;; inside string literals are ignored. -(define comment-lexer - (lexer-srcloc - [racket-line-comment (build-racket-line-comment lexeme)] - [racket-block-comment (build-racket-block-comment lexeme)] - [any-char (return-without-srcloc (comment-lexer input-port))])) - - -(module+ test - (test-case "comment-lexer" - - (define (natural-range start end) - (closed-open-range start end #:comparator natural<=>)) - - (define (read-comments-for-test test-program) - (read-comment-locations (open-input-string test-program 'test-program))) - - (test-case "line comments" - (define input "; This is a comment\n") - (define expected (range-set (natural-range 0 20))) - (check-equal? (read-comments-for-test input) expected)) - - (test-case "double semicolon line comments" - (define input ";; This is a comment\n") - (define expected (range-set (natural-range 0 21))) - (check-equal? (read-comments-for-test input) expected)) - - (test-case "line comments after expressions" - (define input "(void) ; This is a comment\n") - (define expected (range-set (natural-range 7 27))) - (check-equal? (read-comments-for-test input) expected)) - - (test-case "line comments above expressions" - (define input "; This is a comment\n(void)\n") - (define expected (range-set (natural-range 0 20))) - (check-equal? (read-comments-for-test input) expected)) - - (test-case "line comments with non-ASCII characters" - (define input "; λλλλλ\n") - (define expected (range-set (natural-range 0 8))) - (check-equal? (read-comments-for-test input) expected)) - - (test-case "block comments" - (define input "#|\nThis is a block comment\n|#\n") - (define expected (range-set (natural-range 0 29))) - (check-equal? (read-comments-for-test input) expected)) - - (test-case "block comments below expressions" - (define input "(void)\n#|\nThis is a block comment\n|#\n") - (define expected (range-set (natural-range 7 36))) - (check-equal? (read-comments-for-test input) expected)) - - (test-case "block comments above expressions" - (define input "#|\nThis is a block comment\n|#\n(void)\n") - (define expected (range-set (natural-range 0 29))) - (check-equal? (read-comments-for-test input) expected)) - - (test-case "multiple line comments" - (define input "; Line 1\n; Line 2\n; Line 3\n") - (define expected (range-set (natural-range 0 27))) - (check-equal? (read-comments-for-test input) expected)))) diff --git a/private/source.rkt b/private/source.rkt index 1f7ddd47..b68c4907 100644 --- a/private/source.rkt +++ b/private/source.rkt @@ -18,6 +18,7 @@ [source-expand (-> source? syntax?)] [source-can-expand? (-> source? boolean?)] [source-text-of (-> source? syntax? immutable-string?)] + [source-comment-locations (-> source? immutable-range-set?)] [file-source? (-> any/c boolean?)] [file-source (-> path-string? file-source?)] [file-source-path (-> file-source? path?)] @@ -38,7 +39,13 @@ rebellion/base/immutable-string resyntax/private/syntax-neighbors syntax/modread - syntax/parse) + rebellion/base/comparator + rebellion/base/range + rebellion/collection/range-set + rebellion/collection/vector/builder + rebellion/streaming/transducer + syntax-color/lexer-contract + syntax-color/module-lexer) (module+ test @@ -179,3 +186,45 @@ (define start (sub1 (syntax-position stx))) (define end (+ start (syntax-span stx))) (string->immutable-string (substring (source->string code) start end))) + + +(define (source-comment-locations src) + (transduce (source-tokens src) + (filtering lexical-token-comment?) + (mapping lexical-token-location) + #:into (into-range-set natural<=>))) + + +(struct lexical-token (text start end type delimiter-kind attributes) #:transparent) + + +(define (source-tokens src) + (with-input-from-source src + (λ () + (define tokens (make-vector-builder)) + (let loop ([offset 0] [mode #false]) + (define-values (text raw-attributes delimiter-kind start end _ new-mode) + (module-lexer* (current-input-port) offset mode)) + (unless (eof-object? text) + (define type + (if (symbol? raw-attributes) + raw-attributes + (hash-ref raw-attributes 'type))) + (define attributes + (if (symbol? raw-attributes) + (hasheq) + (hash-remove raw-attributes 'type))) + (vector-builder-add tokens (lexical-token text (sub1 start) (sub1 end) type delimiter-kind attributes)) + (loop (sub1 end) (if (dont-stop? new-mode) (dont-stop-val new-mode) new-mode)))) + (build-vector tokens)))) + + +(define (lexical-token-comment? token) + (define type (lexical-token-type token)) + (or (equal? type 'comment) + (equal? type 'sexp-comment) + (hash-ref (lexical-token-attributes token) 'comment? #false))) + + +(define (lexical-token-location token) + (closed-open-range (lexical-token-start token) (lexical-token-end token) #:comparator natural<=>)) diff --git a/test/private/rackunit.rkt b/test/private/rackunit.rkt index 17a9f096..424d4f49 100644 --- a/test/private/rackunit.rkt +++ b/test/private/rackunit.rkt @@ -128,7 +128,7 @@ (define current-line-mask (make-parameter (range-set (unbounded-range #:comparator natural<=>)))) -(define current-analyzer-timeout-millis (make-parameter 100)) +(define current-analyzer-timeout-millis (make-parameter 1000)) (define (range-bound-add bound amount)