Merge pull request #1849 from markshannon/python-add-syntax-comments

Python: Add syntax example comments for automatic document generation.

Author: taus-semmle

python/ql/src/semmle/python/AstExtended.qll

@@ -3,6 +3,11 @@ import python
 /** Syntactic node (Class, Function, Module, Expr, Stmt or Comprehension) corresponding to a flow node */
 abstract class AstNode extends AstNode_ {
 
+    /* Special comment for documentation generation.
+     * All subclasses of `AstNode` that represent concrete syntax should have
+     * a comment of the form: */
+    /* syntax: ... */
+
     /** Gets the scope that this node occurs in */
     abstract Scope getScope();
 
@@ -206,6 +211,8 @@ class ComprehensionList extends ComprehensionList_ {
 /** A list of expressions */
 class ExprList extends ExprList_ {
 
+    /* syntax: Expr, ... */
+
 }
 
 

python/ql/src/semmle/python/Class.qll

@@ -63,6 +63,8 @@ class ClassExpr extends ClassExpr_ {
 /** A class statement. Note that ClassDef extends Assign as a class definition binds the newly created class */
 class ClassDef extends Assign {
 
+    /* syntax: class name(...): ... */
+
     ClassDef() {
         /* This is an artificial assignment the rhs of which is a (possibly decorated) ClassExpr */
         exists(ClassExpr c | this.getValue() = c or this.getValue() = c.getADecoratorCall())

python/ql/src/semmle/python/Exprs.qll

@@ -131,6 +131,8 @@ class Expr extends Expr_, AstNode {
 /** An attribute expression, such as `value.attr` */
 class Attribute extends Attribute_ {
 
+    /* syntax: Expr.name */
+
     override Expr getASubExpression() {
         result = this.getObject()
     }
@@ -160,6 +162,8 @@ class Attribute extends Attribute_ {
 /** A subscript expression, such as `value[slice]` */
 class Subscript extends Subscript_ {
 
+    /* syntax: Expr[Expr] */
+
     override Expr getASubExpression() {
         result = this.getIndex()
         or
@@ -176,6 +180,8 @@ class Subscript extends Subscript_ {
 /** A call expression, such as `func(...)` */
 class Call extends Call_ {
 
+    /* syntax: Expr(...) */
+
     override Expr getASubExpression() {
         result = this.getAPositionalArg() or
         result = this.getAKeyword().getValue() or
@@ -268,6 +274,8 @@ class Call extends Call_ {
 /** A conditional expression such as, `body if test else orelse` */
 class IfExp extends IfExp_ {
 
+    /* syntax: Expr if Expr else Expr */
+
     override Expr getASubExpression() {
         result = this.getTest() or result = this.getBody() or result = this.getOrelse()
     }
@@ -278,6 +286,8 @@ class IfExp extends IfExp_ {
 /** A starred expression, such as the `*rest` in the assignment `first, *rest = seq` */
 class Starred extends Starred_ {
 
+    /* syntax: *Expr */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -288,6 +298,8 @@ class Starred extends Starred_ {
 /** A yield expression, such as `yield value` */
 class Yield extends Yield_ {
 
+    /* syntax: yield Expr */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -301,6 +313,8 @@ class Yield extends Yield_ {
 /** A yield expression, such as `yield from value` */
 class YieldFrom extends YieldFrom_ {
 
+    /* syntax: yield from Expr */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -314,6 +328,8 @@ class YieldFrom extends YieldFrom_ {
 /** A repr (backticks) expression, such as `` `value` `` */
 class Repr extends Repr_ {
 
+    /* syntax: `Expr` */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -330,6 +346,8 @@ class Repr extends Repr_ {
    `"hello"` are treated as Bytes for Python2, but Unicode for Python3. */
 class Bytes extends StrConst {
 
+    /* syntax: b"hello" */
+
     Bytes() {
         not this.isUnicode()
     }
@@ -355,6 +373,8 @@ class Bytes extends StrConst {
 /** An ellipsis expression, such as `...` */
 class Ellipsis extends Ellipsis_ {
 
+    /* syntax: ... */
+
     override Expr getASubExpression() {
         none()
     }
@@ -394,6 +414,8 @@ abstract class Num extends Num_, ImmutableLiteral {
 /** An integer numeric constant, such as `7` or `0x9` */
 class IntegerLiteral extends Num {
 
+    /* syntax: 4 */
+
     IntegerLiteral() {
         not this instanceof FloatLiteral and not this instanceof ImaginaryLiteral
     }
@@ -425,6 +447,8 @@ class IntegerLiteral extends Num {
 /** A floating point numeric constant, such as `0.4` or `4e3` */
 class FloatLiteral extends Num {
 
+    /* syntax: 4.2 */
+
     FloatLiteral() {
         not this instanceof ImaginaryLiteral and
         this.getN().regexpMatch(".*[.eE].*")
@@ -456,6 +480,8 @@ class FloatLiteral extends Num {
 class ImaginaryLiteral extends Num {
     private float value;
 
+    /* syntax: 1.0j */
+
     ImaginaryLiteral() {
         value = this.getN().regexpCapture("(.+)j.*", 1).toFloat()
     }
@@ -513,6 +539,8 @@ class NegativeIntegerLiteral extends ImmutableLiteral, UnaryExpr {
    "hello" are treated as Bytes for Python2, but Unicode for Python3. */
 class Unicode extends StrConst {
 
+    /* syntax: "hello" */
+
     Unicode() {
         this.isUnicode()
     }
@@ -541,6 +569,8 @@ class Unicode extends StrConst {
 /** A dictionary expression, such as `{'key':'value'}` */
 class Dict extends Dict_ {
 
+    /* syntax: {Expr: Expr, ...} */
+
     /** Gets the value of an item of this dict display */
     Expr getAValue() {
         result = this.getAnItem().(DictDisplayItem).getValue()
@@ -566,6 +596,8 @@ class Dict extends Dict_ {
 /** A list expression, such as `[ 1, 3, 5, 7, 9 ]` */
 class List extends List_ {
 
+    /* syntax: [Expr, ...] */
+
     override Expr getASubExpression() {
         result = this.getAnElt()
     }
@@ -575,6 +607,8 @@ class List extends List_ {
 /** A set expression such as `{ 1, 3, 5, 7, 9 }` */
 class Set extends Set_ {
 
+    /* syntax: {Expr, ...} */
+
     override Expr getASubExpression() {
         result = this.getAnElt()
     }
@@ -601,6 +635,8 @@ class PlaceHolder extends PlaceHolder_ {
 /** A tuple expression such as `( 1, 3, 5, 7, 9 )` */
 class Tuple extends Tuple_ {
 
+    /* syntax: (Expr, ...) */
+
     override Expr getASubExpression() {
         result = this.getAnElt()
     }
@@ -612,6 +648,8 @@ class Tuple extends Tuple_ {
  */
 class Name extends Name_ {
 
+    /* syntax: name */
+
     string getId() {
         result = this.getVariable().getId()
     }
@@ -705,6 +743,8 @@ class Slice extends Slice_ {
 /** A string constant. */
 class StrConst extends Str_, ImmutableLiteral {
 
+    /* syntax: "hello" */
+
     predicate isUnicode() {
         this.getPrefix().charAt(_) = "u"
         or
@@ -790,6 +830,8 @@ abstract class BooleanLiteral extends NameConstant {
 /** The boolean named constant `True` */
 class True extends BooleanLiteral {
 
+    /* syntax: True */
+
     True() {
         name_consts(this, "True")
     }
@@ -807,6 +849,8 @@ class True extends BooleanLiteral {
 /** The boolean named constant `False` */
 class False extends BooleanLiteral {
 
+    /* syntax: False */
+
     False() {
         name_consts(this, "False")
     }
@@ -824,6 +868,8 @@ class False extends BooleanLiteral {
 /** `None` */
 class None extends NameConstant {
 
+    /* syntax: None */
+
     None() {
         name_consts(this, "None")
     }
@@ -840,6 +886,8 @@ class None extends NameConstant {
 /** An await expression such as `await coro`. */
 class Await extends Await_ {
 
+    /* syntax: await Expr */
+
     override Expr getASubExpression() {
         result = this.getValue()
     }
@@ -849,6 +897,8 @@ class Await extends Await_ {
 /** A formatted string literal expression, such as `f'hello {world!s}'` */
 class Fstring extends Fstring_ {
 
+    /* syntax: f"Yes!" */
+
     override Expr getASubExpression() {
         result = this.getAValue()
     }

python/ql/src/semmle/python/Function.qll

@@ -187,6 +187,8 @@ class Function extends Function_, Scope, AstNode {
 /** A def statement. Note that FunctionDef extends Assign as a function definition binds the newly created function */
 class FunctionDef extends Assign {
 
+    /* syntax: def name(...): ... */
+
     FunctionDef() {
         /* This is an artificial assignment the rhs of which is a (possibly decorated) FunctionExpr */
         exists(FunctionExpr f | this.getValue() = f or this.getValue() = f.getADecoratorCall())

python/ql/src/semmle/python/Import.qll

@@ -170,6 +170,8 @@ class ImportMember extends ImportMember_ {
 /** An import statement */
 class Import extends Import_ {
 
+    /* syntax: import modname */
+
     private ImportExpr getAModuleExpr() {
         result = this.getAName().getValue()
         or 
@@ -222,6 +224,8 @@ class Import extends Import_ {
 /** An import * statement */
 class ImportStar extends ImportStar_ {
 
+    /* syntax: from modname import * */
+
     ImportExpr getModuleExpr() {
         result = this.getModule()
         or

python/ql/src/semmle/python/Keywords.qll

@@ -2,6 +2,8 @@ import python
 
 class KeyValuePair extends KeyValuePair_, DictDisplayItem {
 
+    /* syntax: Expr : Expr */
+
     override Location getLocation() {
         result = KeyValuePair_.super.getLocation()
     }
@@ -76,6 +78,8 @@ abstract class DictDisplayItem extends DictItem {
 /** A keyword argument in a call. For example `arg=expr` in `foo(0, arg=expr)` */
 class Keyword extends Keyword_, DictUnpackingOrKeyword  {
 
+    /* syntax: name = Expr */
+
     override Location getLocation() {
         result = Keyword_.super.getLocation()
     }