Skip to content

Commit

Permalink
started work on eol comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@robertmuth
robertmuth committed Feb 3, 2024
1 parent b992866 commit 91eb3ee
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 53 deletions.
12 changes: 6 additions & 6 deletions FrontEnd/LangTest/rec_test.cw
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
@pub (defrec type_rec1 :
@doc "this is a comment with \" with quotes \t "
(field s1 s32)
@doc "s2 comment "
(field s2 s32)
(field s3 s32)
(field s3 @eoldoc "s3 is ..." s32)
(field s4 bool)
(field s5 u64)
(field s6 u64))
Expand Down Expand Up @@ -37,7 +38,7 @@
(global u0 u32 0x12345678)


(global g0 type_rec1 undef)
(global g0 type_rec1 @eoldoc "g0 is i mportant" undef)


(global g1 (array 5 type_rec1) undef)
Expand All @@ -62,7 +63,7 @@
(array_val 13 u16 [ 0x11 undef 0x12 ])]))


(global g4 auto (array_val 4 type_rec2 [(index_val undef) (index_val g2)]))
(global g4 auto (array_val 4 type_rec2 [(index_val undef) @eoldoc " BROKEN init" (index_val g2)]))

@pub (defrec type_rec5 :
(field t1 u64)
Expand All @@ -79,7 +80,7 @@

@cdecl (fun main [(param argc s32) (param argv (ptr (ptr u8)))] s32 :
@doc "LOCAL"
(let! v1 auto (rec_val type_rec3 []))
(let! v1 auto @eoldoc "after let " (rec_val type_rec3 []))
(= (. v1 u2) 102)
(= (. v1 u3) 103)
(= (. v1 u6) 106)
Expand Down Expand Up @@ -128,6 +129,5 @@
(test::AssertEq# (at (. g3 u5) 10) 510_u16)
@doc "test end"
(test::Success#)
(return 0))

(return @eoldoc "after return" 0))
)
11 changes: 9 additions & 2 deletions FrontEnd/cwast.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1413,6 +1413,7 @@ class IndexVal:
init_index: "NODES_EXPR_T" # compile time constant
#
doc: str = ""
eoldoc: str = ""
#
x_srcloc: SrcLoc = SRCLOC_UNKNOWN
x_type: CanonType = NO_TYPE
Expand Down Expand Up @@ -1540,6 +1541,8 @@ class ValRec:
inits_field: List[NODES_INITS_REC_T]
#
doc: str = ""
eoldoc: str = ""

#
x_srcloc: SrcLoc = SRCLOC_UNKNOWN
x_type: CanonType = NO_TYPE
Expand Down Expand Up @@ -2281,6 +2284,7 @@ class StmtReturn:
expr_ret: NODES_EXPR_T
#
doc: str = ""
eoldoc: str = ""
#
x_srcloc: SrcLoc = SRCLOC_UNKNOWN
x_target: Optional[Any] = None
Expand Down Expand Up @@ -2361,6 +2365,7 @@ class StmtCompoundAssignment:
expr_rhs: NODES_EXPR_T
#
doc: str = ""
eoldoc: str = ""
#
x_srcloc: SrcLoc = SRCLOC_UNKNOWN

Expand All @@ -2380,6 +2385,7 @@ class StmtAssignment:
expr_rhs: NODES_EXPR_T
#
doc: str = ""
eoldoc: str = ""
#
x_srcloc: SrcLoc = SRCLOC_UNKNOWN

Expand All @@ -2406,6 +2412,7 @@ class RecField: #
type: NODES_TYPES_T
#
doc: str = ""
eoldoc: str = ""
#
x_srcloc: SrcLoc = SRCLOC_UNKNOWN
x_type: CanonType = NO_TYPE
Expand Down Expand Up @@ -3290,9 +3297,9 @@ def GenerateDocumentation(fout):

print("## Enum Details", file=fout)

_RenderKind(Expr1.__name__, UNARY_EXPR_KIND,
_RenderKind(Expr1.__name__, UNARY_EXPR_KIND,
UNARY_EXPR_SHORTCUT_INV, fout)
_RenderKind(Expr2.__name__, BINARY_EXPR_KIND,
_RenderKind(Expr2.__name__, BINARY_EXPR_KIND,
BINARY_EXPR_SHORTCUT_INV, fout)
_RenderKind(ExprPointer.__name__, POINTER_EXPR_KIND,
POINTER_EXPR_SHORTCUT_INV, fout)
Expand Down
81 changes: 43 additions & 38 deletions FrontEnd/parse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import re
import logging

from typing import List, Any, Dict, Tuple
from typing import List, Any, Dict, Tuple, Union

from FrontEnd import cwast

Expand Down Expand Up @@ -52,14 +52,12 @@
_RE_TOKEN_NUM = re.compile(r'-?[.0-9][-+_.a-z0-9]*')





def ReadAttrs(t: str, attr, stream):
def ReadAttrs(t: str, attr: Dict[str, Any], stream):
"""attr is indended to be used for node creation as a **attr parameter."""
while t.startswith("@"):
tag = t[1:]
val = True
if tag == "doc":
val: Union[bool, str] = True
if tag in ("doc", "eoldoc"):
val = next(stream)
attr[tag] = val
t = next(stream)
Expand Down Expand Up @@ -141,25 +139,27 @@ def __next__(self):


def _MakeTypeBaseLambda(kind: cwast.BASE_TYPE_KIND):
return lambda srcloc: cwast.TypeBase(kind, x_srcloc=srcloc)
def closure(**extra):
return cwast.TypeBase(kind, **extra)
return closure


# maps "atoms" to the nodes they will be expanded to
_SHORT_HAND_NODES = {
"auto": lambda srcloc: cwast. TypeAuto(x_srcloc=srcloc),
"auto": cwast.TypeAuto,
#
"noret": _MakeTypeBaseLambda(cwast.BASE_TYPE_KIND.NORET),
"bool": _MakeTypeBaseLambda(cwast.BASE_TYPE_KIND.BOOL),
"void": _MakeTypeBaseLambda(cwast.BASE_TYPE_KIND.VOID),
#
"auto_val": lambda srcloc: cwast.ValAuto(x_srcloc=srcloc),
"void_val": lambda srcloc: cwast.ValVoid(x_srcloc=srcloc),
"undef": lambda srcloc: cwast.ValUndef(x_srcloc=srcloc),
"true": lambda srcloc: cwast.ValTrue(x_srcloc=srcloc),
"false": lambda srcloc: cwast.ValFalse(x_srcloc=srcloc),
"auto_val": cwast.ValAuto,
"void_val": cwast.ValVoid,
"undef": cwast.ValUndef,
"true": cwast.ValTrue,
"false": cwast.ValFalse,
# see cwast.OPTIONAL_FIELDS
"break": lambda srcloc: cwast.StmtBreak(target="", x_srcloc=srcloc),
"continue": lambda srcloc: cwast.StmtContinue(target="", x_srcloc=srcloc),
"break": lambda args: cwast.StmtBreak(target="", **args),
"continue": lambda args: cwast.StmtContinue(target="", **args),
}

# add basic type names
Expand All @@ -181,11 +181,12 @@ def IsWellFormedStringLiteral(t: str):
return False


def ExpandShortHand(t: str, srcloc) -> Any:
def ExpandShortHand(t: str, srcloc, attr: Dict[str, Any]) -> Any:
"""Expands atoms, ids, and numbers to proper nodes"""
x = _SHORT_HAND_NODES.get(t)
if x is not None:
return x(srcloc)
node = x(x_srcloc=srcloc, **attr)
return node

if IsWellFormedStringLiteral(t):
logger.info("STRING %s at %s", t, srcloc)
Expand All @@ -207,20 +208,20 @@ def ExpandShortHand(t: str, srcloc) -> Any:
t = t[1:-1]

return cwast.ValString(t, x_srcloc=srcloc, strkind=strkind,
triplequoted=triplequoted)
triplequoted=triplequoted, **attr)
elif _RE_TOKEN_ID.fullmatch(t):
if t in cwast.NODES_ALIASES:
cwast.CompilerError(srcloc, f"Reserved name used as ID: {t}")
if t[0] == "$":
return cwast.MacroId(t, x_srcloc=srcloc)
logger.info("ID %s at %s", t, srcloc)
return cwast.Id(t, x_srcloc=srcloc)
return cwast.Id(t, x_srcloc=srcloc, **attr)
elif _RE_TOKEN_NUM.fullmatch(t):
logger.info("NUM %s at %s", t, srcloc)
return cwast.ValNum(t, x_srcloc=srcloc)
return cwast.ValNum(t, x_srcloc=srcloc, **attr)
elif len(t) >= 2 and t[0] == "'" and t[-1] == "'":
logger.info("CHAR %s at %s", t, srcloc)
return cwast.ValNum(t, x_srcloc=srcloc)
return cwast.ValNum(t, x_srcloc=srcloc, **attr)
else:
cwast.CompilerError(srcloc, f"unexpected token {repr(t)}")

Expand All @@ -236,15 +237,17 @@ def ReadNodeList(stream: ReadTokens, parent_cls) -> List[Any]:
if token == "(":
expr = ReadSExpr(stream, parent_cls, attr)
else:
expr = ExpandShortHand(token, stream.srcloc())
# hack for simpler array val and rec val initializers
expr = ExpandShortHand(token, stream.srcloc(), attr)
attr.clear()
# hack for simpler array val and rec val initializers: take the expr
# from above and wrap it into a IndexVal or FieldVal
if parent_cls is cwast.ValArray and not isinstance(expr, cwast.IndexVal):
expr = cwast.IndexVal(expr, cwast.ValAuto(
x_srcloc=expr.x_srcloc), x_srcloc=expr.x_srcloc, **attr)
x_srcloc=expr.x_srcloc), x_srcloc=expr.x_srcloc)
elif parent_cls is cwast.ValRec and not isinstance(expr, cwast.FieldVal):
expr = cwast.FieldVal(expr, "", x_srcloc=expr.x_srcloc, **attr)
expr = cwast.FieldVal(expr, "", x_srcloc=expr.x_srcloc)
out.append(expr)
attr.clear()

return out


Expand All @@ -265,10 +268,7 @@ def ReadNodeColonList(stream: ReadTokens, parent_cls):
if token == "(":
expr = ReadSExpr(stream, parent_cls, attr)
else:
if attr:
cwast.CompilerError(
stream.srcloc(), f"unexpected attribs: {attr}")
expr = ExpandShortHand(token, stream.srcloc())
expr = ExpandShortHand(token, stream.srcloc(), attr)
out.append(expr)
attr.clear()

Expand Down Expand Up @@ -313,11 +313,11 @@ def ReadPiece(field, token, stream: ReadTokens, parent_cls) -> Any:
cwast.CompilerError(
stream.srcloc(), f"Cannot convert {token} for {field}")
elif nfd.kind is cwast.NFK.NODE:
attr = {}
token = ReadAttrs(token, {}, stream)
attr: Dict[str, Any] = {}
token = ReadAttrs(token, attr, stream)
if token == "(":
return ReadSExpr(stream, parent_cls, attr)
out = ExpandShortHand(token, stream.srcloc())
out = ExpandShortHand(token, stream.srcloc(), attr)
if out is None:
cwast.CompilerError(
stream.srcloc(), f"Cannot expand {token} for {field}")
Expand All @@ -343,25 +343,30 @@ def ReadPiece(field, token, stream: ReadTokens, parent_cls) -> Any:
assert None


def ReadMacroInvocation(tag, stream: ReadTokens):
def ReadMacroInvocation(tag: str, stream: ReadTokens):
"""The leading '(' and `tag` have already been consumed"""
parent_cls = cwast.MacroInvoke
srcloc = stream.srcloc()
logger.info("Readdng MACRO INVOCATION %s at %s", tag, srcloc)
args: List[Any] = []
while True:
token = next(stream)
attr = {}
token = ReadAttrs(token, {}, stream)
if token == ")":
return cwast.MacroInvoke(tag, args, x_srcloc=srcloc)
return cwast.MacroInvoke(tag, args, x_srcloc=srcloc, **attr)
elif token == "(":
args.append(ReadSExpr(stream, parent_cls, {}))
args.append(ReadSExpr(stream, parent_cls, attr))
elif token == "[":
assert not attr
args.append(cwast.EphemeralList(ReadNodeList(
stream, parent_cls), colon=False, x_srcloc=srcloc))
elif token == ":":
assert not attr
args.append(cwast.EphemeralList(ReadNodeColonList(
stream, parent_cls), colon=True, x_srcloc=srcloc))
else:
out = ExpandShortHand(token, stream.srcloc())
out = ExpandShortHand(token, stream.srcloc(), attr)
assert out is not None, f"while processing {tag} unexpected macro arg: {token}"
args.append(out)
return args
Expand Down
31 changes: 24 additions & 7 deletions FrontEnd/pp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -591,7 +591,7 @@ def TokensFunctional(ts: TS, name, nodes: List):

def TokensBinaryInfix(ts: TS, name: str, node1, node2, node):
EmitTokens(ts, node1)
TokensAnnotations(ts, node)
TokensAnnotationsPre(ts, node)
if name in (".", "->"):
ts.EmitBinOpNoSpace(name)
else:
Expand All @@ -615,15 +615,17 @@ def EmitExpr3(ts: TS, node: cwast.Expr3):
EmitTokens(ts, node.expr_f)


def TokensAnnotations(ts: TS, node):
def TokensAnnotationsPre(ts: TS, node):
# handle docs first
for field, nfd in node.ATTRS:
# these attributes will be rendered directly
if nfd.kind is not cwast.NFK.ATTR_STR:
continue
val = getattr(node, field)
if val:
if field == "doc":
if field == "eoldoc":
continue
elif field == "doc":
if val.startswith('"""'):
val = val[3:-3]
else:
Expand All @@ -645,6 +647,20 @@ def TokensAnnotations(ts: TS, node):
ts.EmitAnnotationShort("@" + field)


def TokensAnnotationsPost(ts: TS, node):
for field, nfd in node.ATTRS:
# these attributes will be rendered directly
if field != "eoldoc":
continue
val = getattr(node, field)
if val:
if val.startswith('"""'):
val = val[3:-3]
else:
val = val[1:-1]
ts.EmitComment(" -- " + val)


def TokensMacroInvoke(ts: TS, node: cwast.MacroInvoke):
if node.name == "->":
assert len(node.args) == 2
Expand Down Expand Up @@ -809,9 +825,11 @@ def TokensDefMod(ts: TS, node: cwast.DefMod):
ts.EmitEnd(beg_colon)
ts.EmitEnd(beg)


def WithMut(name: str, mutable: bool) -> str:
return name + "!" if mutable else name


def TokensDefGlobal(ts: TS, node: cwast.DefGlobal):
beg = ts.EmitBeg(WithMut("global", node.mut))
ts.EmitAttr(node.name)
Expand Down Expand Up @@ -961,9 +979,6 @@ def TokensMacroId(ts: TS, node: cwast.MacroId):
])





_CONCRETE_SYNTAX = {
cwast.Id: lambda ts, n: (ts.EmitAttr(n.name)),
#
Expand Down Expand Up @@ -1056,11 +1071,13 @@ def TokensMacroId(ts: TS, node: cwast.MacroId):

def EmitTokens(ts: TS, node):
if node.__class__ not in _INFIX_OPS:
TokensAnnotations(ts, node)
TokensAnnotationsPre(ts, node)

gen = _CONCRETE_SYNTAX.get(node.__class__)
assert gen, f"unknown node {node.__class__}"
gen(ts, node)
if node.__class__ not in _INFIX_OPS:
TokensAnnotationsPost(ts, node)


class Stack:
Expand Down

0 comments on commit 91eb3ee

Please sign in to comment.