From 08b82c90f520180f6a9cba6c816b6d4a5a4e4424 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 2 Nov 2024 00:01:45 +0800 Subject: [PATCH 1/5] improve httpclient docuementation (#24398) ref https://github.com/nim-lang/Nim/issues/24394 --- lib/pure/httpclient.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 08ea99627c1c..654bed27bc91 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -62,6 +62,7 @@ ## validated to the server. ## ## ```Nim +## import std/[httpclient] ## var client = newHttpClient() ## var data = newMultipartData() ## data["output"] = "soap12" @@ -79,6 +80,7 @@ ## it, you can pass your own via the `mimeDb` parameter to avoid this. ## ## ```Nim +## import std/[httpclient, mimetypes] ## let mimes = newMimetypes() ## var client = newHttpClient() ## var data = newMultipartData() @@ -160,7 +162,7 @@ ## Example of setting SSL verification parameters in a new client: ## ## ```Nim -## import httpclient +## import std/[net, httpclient] ## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) ## ``` ## From 46bb47a444bd377860d832fc1c62b262343f36a2 Mon Sep 17 00:00:00 2001 From: Phil Krylov Date: Sat, 2 Nov 2024 07:58:19 +0100 Subject: [PATCH 2/5] std/parsesql: Fix JOIN parsing (#22890) This commit fixes/adds tests for and fixes several issues with `JOIN` operator parsing: - For OUTER joins, LEFT | RIGHT | FULL specifier is not optional ```nim doAssertRaises(SqlParseError): discard parseSql(""" SELECT id FROM a OUTER JOIN b ON a.id = b.id """) ``` - For NATURAL JOIN and CROSS JOIN, ON and USING clauses are forbidden ```nim doAssertRaises(SqlParseError): discard parseSql(""" SELECT id FROM a CROSS JOIN b ON a.id = b.id """) ``` - JOIN should parse as part of FROM, not after WHERE ```nim doAssertRaises(SqlParseError): discard parseSql(""" SELECT id FROM a WHERE a.id IS NOT NULL INNER JOIN b ON a.id = b.id """) ``` - LEFT JOIN should parse ```nim doAssert $parseSql(""" SELECT id FROM a LEFT JOIN b ON a.id = b.id """) == "select id from a left join b on a.id = b.id;" ``` - NATURAL JOIN should parse ```nim doAssert $parseSql(""" SELECT id FROM a NATURAL JOIN b """) == "select id from a natural join b;" ``` - USING should parse ```nim doAssert $parseSql(""" SELECT id FROM a JOIN b USING (id) """) == "select id from a join b using (id );" ``` - Multiple JOINs should parse ```nim doAssert $parseSql(""" SELECT id FROM a JOIN b ON a.id = b.id LEFT JOIN c USING (id) """) == "select id from a join b on a.id = b.id left join c using (id );" ``` --- lib/pure/parsesql.nim | 114 ++++++++++++++++++++++++++++--------- tests/stdlib/tparsesql.nim | 71 +++++++++++++++++++++-- 2 files changed, 152 insertions(+), 33 deletions(-) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index a7c938d0181a..4ea01c89d8de 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -507,12 +507,14 @@ type nkAsgn, nkFrom, nkFromItemPair, + nkJoin, + nkNaturalJoin, + nkUsing, nkGroup, nkLimit, nkOffset, nkHaving, nkOrder, - nkJoin, nkDesc, nkUnion, nkIntersect, @@ -936,18 +938,75 @@ proc parseWhere(p: var SqlParser): SqlNode = result = newNode(nkWhere) result.add(parseExpr(p)) +proc parseJoinType(p: var SqlParser): SqlNode = + ## parse [ INNER ] JOIN | ( LEFT | RIGHT | FULL ) [ OUTER ] JOIN + if isKeyw(p, "inner"): + getTok(p) + eat(p, "join") + return newNode(nkIdent, "inner") + elif isKeyw(p, "join"): + getTok(p) + return newNode(nkIdent, "") + elif isKeyw(p, "left") or isKeyw(p, "full") or isKeyw(p, "right"): + var joinType = newNode(nkIdent, p.tok.literal.toLowerAscii()) + getTok(p) + optKeyw(p, "outer") + eat(p, "join") + return joinType + else: + sqlError(p, "join type expected") + proc parseFromItem(p: var SqlParser): SqlNode = result = newNode(nkFromItemPair) + var expectAs = true if p.tok.kind == tkParLe: getTok(p) - var select = parseSelect(p) - result.add(select) + if isKeyw(p, "select"): + result.add(parseSelect(p)) + else: + result = parseFromItem(p) + expectAs = false eat(p, tkParRi) else: result.add(parseExpr(p)) - if isKeyw(p, "as"): + if expectAs and isKeyw(p, "as"): getTok(p) result.add(parseExpr(p)) + while true: + if isKeyw(p, "cross"): + var join = newNode(nkJoin) + join.add(newNode(nkIdent, "cross")) + join.add(result) + getTok(p) + eat(p, "join") + join.add(parseFromItem(p)) + result = join + elif isKeyw(p, "natural"): + var join = newNode(nkNaturalJoin) + getTok(p) + join.add(parseJoinType(p)) + join.add(result) + join.add(parseFromItem(p)) + result = join + elif isKeyw(p, "inner") or isKeyw(p, "join") or isKeyw(p, "left") or + iskeyw(p, "full") or isKeyw(p, "right"): + var join = newNode(nkJoin) + join.add(parseJoinType(p)) + join.add(result) + join.add(parseFromItem(p)) + if isKeyw(p, "on"): + getTok(p) + join.add(parseExpr(p)) + elif isKeyw(p, "using"): + getTok(p) + var n = newNode(nkUsing) + parseParIdentList(p, n) + join.add n + else: + sqlError(p, "ON or USING expected") + result = join + else: + break proc parseIndexDef(p: var SqlParser): SqlNode = result = parseIfNotExists(p, nkCreateIndex) @@ -1109,19 +1168,6 @@ proc parseSelect(p: var SqlParser): SqlNode = elif isKeyw(p, "except"): result.add(newNode(nkExcept)) getTok(p) - if isKeyw(p, "join") or isKeyw(p, "inner") or isKeyw(p, "outer") or isKeyw(p, "cross"): - var join = newNode(nkJoin) - result.add(join) - if isKeyw(p, "join"): - join.add(newNode(nkIdent, "")) - getTok(p) - else: - join.add(newNode(nkIdent, p.tok.literal.toLowerAscii())) - getTok(p) - eat(p, "join") - join.add(parseFromItem(p)) - eat(p, "on") - join.add(parseExpr(p)) if isKeyw(p, "limit"): getTok(p) var l = newNode(nkLimit) @@ -1388,6 +1434,30 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkFrom: s.addKeyw("from") s.addMulti(n) + of nkJoin, nkNaturalJoin: + var joinType = n.sons[0].strVal + if joinType == "": + joinType = "join" + else: + joinType &= " " & "join" + if n.kind == nkNaturalJoin: + joinType = "natural " & joinType + ra(n.sons[1], s) + s.addKeyw(joinType) + # If the right part of the join is not leaf, parenthesize it + if n.sons[2].kind != nkFromItemPair: + s.add('(') + ra(n.sons[2], s) + s.add(')') + else: + ra(n.sons[2], s) + if n.sons.len > 3: + if n.sons[3].kind != nkUsing: + s.addKeyw("on") + ra(n.sons[3], s) + of nkUsing: + s.addKeyw("using") + rs(n, s) of nkGroup: s.addKeyw("group by") s.addMulti(n) @@ -1403,16 +1473,6 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkOrder: s.addKeyw("order by") s.addMulti(n) - of nkJoin: - var joinType = n.sons[0].strVal - if joinType == "": - joinType = "join" - else: - joinType &= " " & "join" - s.addKeyw(joinType) - ra(n.sons[1], s) - s.addKeyw("on") - ra(n.sons[2], s) of nkDesc: ra(n.sons[0], s) s.addKeyw("desc") diff --git a/tests/stdlib/tparsesql.nim b/tests/stdlib/tparsesql.nim index cd582551df03..269ff7f8b1f6 100644 --- a/tests/stdlib/tparsesql.nim +++ b/tests/stdlib/tparsesql.nim @@ -159,17 +159,76 @@ INNER JOIN b ON a.id == b.id """) == "select id from a inner join b on a.id == b.id;" -doAssert $parseSql(""" +# For OUTER joins, LEFT | RIGHT | FULL specifier is not optional +doAssertRaises(SqlParseError): discard parseSql(""" SELECT id FROM a OUTER JOIN b -ON a.id == b.id -""") == "select id from a outer join b on a.id == b.id;" +ON a.id = b.id +""") -doAssert $parseSql(""" +# For NATURAL JOIN and CROSS JOIN, ON and USING clauses are forbidden +doAssertRaises(SqlParseError): discard parseSql(""" SELECT id FROM a CROSS JOIN b -ON a.id == b.id -""") == "select id from a cross join b on a.id == b.id;" +ON a.id = b.id +""") + +# JOIN should parse as part of FROM, not after WHERE +doAssertRaises(SqlParseError): discard parseSql(""" +SELECT id FROM a +WHERE a.id IS NOT NULL +INNER JOIN b +ON a.id = b.id +""") + +# JOIN should parse as part of FROM, other fromItems may follow +doAssert $parseSql(""" +SELECT id +FROM + a JOIN b ON a.id = b.id, + c +""") == "select id from a join b on a.id = b.id, c;" + +# LEFT JOIN should parse +doAssert $parseSql(""" +SELECT id FROM a +LEFT JOIN b +ON a.id = b.id +""") == "select id from a left join b on a.id = b.id;" + +# NATURAL JOIN should parse +doAssert $parseSql(""" +SELECT id FROM a +NATURAL JOIN b +""") == "select id from a natural join b;" + +# USING should parse +doAssert $parseSql(""" +SELECT id FROM a +JOIN b +USING (id) +""") == "select id from a join b using (id );" + +# Multiple JOINs should parse +doAssert $parseSql(""" +SELECT id FROM a +JOIN b +ON a.id = b.id +LEFT JOIN c +USING (id) +""") == "select id from a join b on a.id = b.id left join c using (id );" + +# Parenthesized JOIN expressions should parse +doAssert $parseSql(""" +SELECT id +FROM a JOIN (b JOIN c USING (id)) ON a.id = b.id +""") == "select id from a join(b join c using (id )) on a.id = b.id;" + +# Left-side parenthesized JOIN expressions should parse +doAssert $parseSql(""" +SELECT id +FROM (b JOIN c USING (id)) JOIN a ON a.id = b.id +""") == "select id from b join c using (id ) join a on a.id = b.id;" doAssert $parseSql(""" CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); From 5f056f87b265f3062588d14792d0f0ff80d8db9a Mon Sep 17 00:00:00 2001 From: metagn Date: Sun, 3 Nov 2024 16:56:20 +0300 Subject: [PATCH 3/5] disable all badssl tests indefinitely (#24403) Flaky not just due to recent ubuntu 24/GCC 14 upgrades, windows fails as well, assuming the issue is with badssl or it's just not worth testing here. --- tests/untestable/thttpclient_ssl_disabled.nim | 3 ++- tests/untestable/thttpclient_ssl_env_var.nim | 14 +++++++------- tests/untestable/thttpclient_ssl_remotenetwork.nim | 5 +++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/untestable/thttpclient_ssl_disabled.nim b/tests/untestable/thttpclient_ssl_disabled.nim index b95dad2c64d5..e3d214763ed7 100644 --- a/tests/untestable/thttpclient_ssl_disabled.nim +++ b/tests/untestable/thttpclient_ssl_disabled.nim @@ -9,7 +9,8 @@ ## nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimDisableCertificateValidation -d:ssl -p:. tests/untestable/thttpclient_ssl_disabled.nim from stdtest/testutils import enableRemoteNetworking -when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(openbsd)): +# badssl tests disabled indefinitely +when false and enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(openbsd)): import httpclient, net, unittest const expired = "https://expired.badssl.com/" diff --git a/tests/untestable/thttpclient_ssl_env_var.nim b/tests/untestable/thttpclient_ssl_env_var.nim index 3f25a6ff4c08..5c99dc74e638 100644 --- a/tests/untestable/thttpclient_ssl_env_var.nim +++ b/tests/untestable/thttpclient_ssl_env_var.nim @@ -19,7 +19,6 @@ from net import newSocket, newContext, wrapSocket, connect, close, Port, from strutils import contains const - expired = "https://expired.badssl.com/" good = "https://google.com/" @@ -56,12 +55,13 @@ suite "SSL certificate check": var ctx = newContext(verifyMode=CVerifyPeerUseEnvVars) ctx.wrapSocket(sock) checkpoint("Socket created") - try: - sock.connect("expired.badssl.com", 443.Port) - fail() - except: - sock.close - check getCurrentExceptionMsg().contains("certificate verify failed") + when false: # badssl tests disabled indefinitely + try: + sock.connect("expired.badssl.com", 443.Port) + fail() + except: + sock.close + check getCurrentExceptionMsg().contains("certificate verify failed") elif existsEnv("SSL_CERT_DIR"): var sock = newSocket() diff --git a/tests/untestable/thttpclient_ssl_remotenetwork.nim b/tests/untestable/thttpclient_ssl_remotenetwork.nim index 3cb759516293..5603154897b0 100644 --- a/tests/untestable/thttpclient_ssl_remotenetwork.nim +++ b/tests/untestable/thttpclient_ssl_remotenetwork.nim @@ -33,7 +33,8 @@ when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(win CertTest = tuple[url:string, category:Category, desc: string] # badssl certs sometimes expire, set to false when that happens - when true: + # badssl now disabled indefinitely + when false: const certificate_tests: array[0..54, CertTest] = [ ("https://wrong.host.badssl.com/", bad, "wrong.host"), ("https://captive-portal.badssl.com/", bad, "captive-portal"), @@ -197,7 +198,7 @@ when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(win type NetSocketTest = tuple[hostname: string, port: Port, category:Category, desc: string] # badssl certs sometimes expire, set to false when that happens - when true: + when false: const net_tests:array[0..3, NetSocketTest] = [ ("imap.gmail.com", 993.Port, good, "IMAP"), ("wrong.host.badssl.com", 443.Port, bad, "wrong.host"), From f5d80ede8015960d39799902e0b0f5c197cb93bd Mon Sep 17 00:00:00 2001 From: metagn Date: Sun, 3 Nov 2024 16:59:50 +0300 Subject: [PATCH 4/5] cbuilder: make Builder an object (#24401) Doing this early is useful so we can move the indentation logic into `Builder` itself rather than mix it with the block logic in `ccgstmts` (the `if` statements in #24381 have not been indented properly either). However it also means `Builder` is now used for code that still generates raw C code, so the diff won't be as clean when these get updated. --- compiler/cbuilderbase.nim | 68 ++++++----- compiler/cbuilderdecls.nim | 20 +++- compiler/ccgcalls.nim | 3 +- compiler/ccgexprs.nim | 227 ++++++++++++++++--------------------- compiler/ccgliterals.nim | 23 ++-- compiler/ccgstmts.nim | 42 ++++--- compiler/ccgthreadvars.nim | 8 +- compiler/ccgtrav.nim | 14 +-- compiler/ccgtypes.nim | 167 +++++++++++++-------------- compiler/cgen.nim | 195 ++++++++++++++++--------------- compiler/cgendata.nim | 14 +-- 11 files changed, 387 insertions(+), 394 deletions(-) diff --git a/compiler/cbuilderbase.nim b/compiler/cbuilderbase.nim index 19f3914d41f5..d6b71645fcb6 100644 --- a/compiler/cbuilderbase.nim +++ b/compiler/cbuilderbase.nim @@ -1,44 +1,56 @@ +import ropes, int128 + type - Snippet = string - Builder = string + Snippet* = string + Builder* = object + buf*: string + +template newBuilder*(s: string): Builder = + Builder(buf: s) + +proc extract*(builder: Builder): Snippet = + builder.buf + +proc add*(builder: var Builder, s: string) = + builder.buf.add(s) -template newBuilder(s: string): Builder = - s +proc add*(builder: var Builder, s: char) = + builder.buf.add(s) -proc addIntValue(builder: var Builder, val: int) = - builder.addInt(val) +proc addIntValue*(builder: var Builder, val: int) = + builder.buf.addInt(val) -proc addIntValue(builder: var Builder, val: int64) = - builder.addInt(val) +proc addIntValue*(builder: var Builder, val: int64) = + builder.buf.addInt(val) -proc addIntValue(builder: var Builder, val: uint64) = - builder.addInt(val) +proc addIntValue*(builder: var Builder, val: uint64) = + builder.buf.addInt(val) -proc addIntValue(builder: var Builder, val: Int128) = - builder.addInt128(val) +proc addIntValue*(builder: var Builder, val: Int128) = + builder.buf.addInt128(val) -template cIntValue(val: int): Snippet = $val -template cIntValue(val: int64): Snippet = $val -template cIntValue(val: uint64): Snippet = $val -template cIntValue(val: Int128): Snippet = $val +template cIntValue*(val: int): Snippet = $val +template cIntValue*(val: int64): Snippet = $val +template cIntValue*(val: uint64): Snippet = $val +template cIntValue*(val: Int128): Snippet = $val import std/formatfloat -proc addFloatValue(builder: var Builder, val: float) = - builder.addFloat(val) +proc addFloatValue*(builder: var Builder, val: float) = + builder.buf.addFloat(val) -template cFloatValue(val: float): Snippet = $val +template cFloatValue*(val: float): Snippet = $val -proc int64Literal(i: BiggestInt; result: var Builder) = +proc addInt64Literal*(result: var Builder; i: BiggestInt) = if i > low(int64): result.add "IL64($1)" % [rope(i)] else: result.add "(IL64(-9223372036854775807) - IL64(1))" -proc uint64Literal(i: uint64; result: var Builder) = +proc addUint64Literal*(result: var Builder; i: uint64) = result.add rope($i & "ULL") -proc intLiteral(i: BiggestInt; result: var Builder) = +proc addIntLiteral*(result: var Builder; i: BiggestInt) = if i > low(int32) and i <= high(int32): result.addIntValue(i) elif i == low(int32): @@ -49,19 +61,19 @@ proc intLiteral(i: BiggestInt; result: var Builder) = else: result.add "(IL64(-9223372036854775807) - IL64(1))" -proc intLiteral(i: Int128; result: var Builder) = - intLiteral(toInt64(i), result) +proc addIntLiteral*(result: var Builder; i: Int128) = + addIntLiteral(result, toInt64(i)) -proc cInt64Literal(i: BiggestInt): Snippet = +proc cInt64Literal*(i: BiggestInt): Snippet = if i > low(int64): result = "IL64($1)" % [rope(i)] else: result = "(IL64(-9223372036854775807) - IL64(1))" -proc cUint64Literal(i: uint64): Snippet = +proc cUint64Literal*(i: uint64): Snippet = result = $i & "ULL" -proc cIntLiteral(i: BiggestInt): Snippet = +proc cIntLiteral*(i: BiggestInt): Snippet = if i > low(int32) and i <= high(int32): result = rope(i) elif i == low(int32): @@ -72,5 +84,5 @@ proc cIntLiteral(i: BiggestInt): Snippet = else: result = "(IL64(-9223372036854775807) - IL64(1))" -proc cIntLiteral(i: Int128): Snippet = +proc cIntLiteral*(i: Int128): Snippet = result = cIntLiteral(toInt64(i)) diff --git a/compiler/cbuilderdecls.nim b/compiler/cbuilderdecls.nim index d717c6ee2e90..70d7cc364fb3 100644 --- a/compiler/cbuilderdecls.nim +++ b/compiler/cbuilderdecls.nim @@ -42,6 +42,18 @@ template addVarWithType(builder: var Builder, kind: VarKind = Local, name: strin builder.add(name) builder.add(";\n") +template addVarWithInitializer(builder: var Builder, kind: VarKind = Local, name: string, + typ: Snippet, initializerBody: typed) = + ## adds a variable declaration to the builder, with + ## `initializerBody` building the initializer. initializer must be provided + builder.addVarHeader(kind) + builder.add(typ) + builder.add(" ") + builder.add(name) + builder.add(" = ") + initializerBody + builder.add(";\n") + template addVarWithTypeAndInitializer(builder: var Builder, kind: VarKind = Local, name: string, typeBody, initializerBody: typed) = ## adds a variable declaration to the builder, with `typeBody` building the type, and @@ -252,12 +264,12 @@ proc startSimpleStruct(obj: var Builder; m: BModule; name: string; baseType: Sni obj.add(baseType) obj.add(" ") obj.add("{\n") - result.preFieldsLen = obj.len + result.preFieldsLen = obj.buf.len if result.baseKind == bcSupField: obj.addField(name = "Sup", typ = baseType) proc finishSimpleStruct(obj: var Builder; m: BModule; info: StructBuilderInfo) = - if info.baseKind == bcNone and info.preFieldsLen == obj.len: + if info.baseKind == bcNone and info.preFieldsLen == obj.buf.len: # no fields were added, add dummy field obj.addField(name = "dummy", typ = "char") if info.named: @@ -308,7 +320,7 @@ proc startStruct(obj: var Builder; m: BModule; t: PType; name: string; baseType: obj.add(baseType) obj.add(" ") obj.add("{\n") - result.preFieldsLen = obj.len + result.preFieldsLen = obj.buf.len case result.baseKind of bcNone: # rest of the options add a field or don't need it due to inheritance, @@ -328,7 +340,7 @@ proc startStruct(obj: var Builder; m: BModule; t: PType; name: string; baseType: obj.addField(name = "Sup", typ = baseType) proc finishStruct(obj: var Builder; m: BModule; t: PType; info: StructBuilderInfo) = - if info.baseKind == bcNone and info.preFieldsLen == obj.len and + if info.baseKind == bcNone and info.preFieldsLen == obj.buf.len and t.itemId notin m.g.graph.memberProcsPerType: # no fields were added, add dummy field obj.addField(name = "dummy", typ = "char") diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 788f7a470bde..5ac567c0b265 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -209,8 +209,7 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareF result = ("($3*)(($1)+($2))" % [rdLoc(a), rdLoc(b), dest], lengthExpr) else: - var lit = newRopeAppender() - intLiteral(first, lit) + let lit = cIntLiteral(first) result = ("($4*)($1)+(($2)-($3))" % [rdLoc(a), rdLoc(b), lit, dest], lengthExpr) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 78e9e5e280cd..1be7f2dc2b13 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -34,15 +34,15 @@ proc genLiteral(p: BProc, n: PNode, ty: PType; result: var Builder) = else: k = tyNil # don't go into the case variant that uses 'ty' case k of tyChar, tyNil: - intLiteral(n.intVal, result) + result.addIntLiteral(n.intVal) of tyBool: if n.intVal != 0: result.add "NIM_TRUE" else: result.add "NIM_FALSE" - of tyInt64: int64Literal(n.intVal, result) - of tyUInt64: uint64Literal(uint64(n.intVal), result) + of tyInt64: result.addInt64Literal(n.intVal) + of tyUInt64: result.addUint64Literal(uint64(n.intVal)) else: result.addCast(getTypeDesc(p.module, ty)): - intLiteral(n.intVal, result) + result.addIntLiteral(n.intVal) of nkNilLit: let k = if ty == nil: tyPointer else: skipTypes(ty, abstractVarRange).kind if k == tyProc and skipTypes(ty, abstractVarRange).callConv == ccClosure: @@ -51,17 +51,14 @@ proc genLiteral(p: BProc, n: PNode, ty: PType; result: var Builder) = if id == p.module.labels: # not found in cache: inc(p.module.labels) - var data = newBuilder("") - data.addVarWithTypeAndInitializer(kind = Const, name = tmpName): - data.add(getTypeDesc(p.module, ty)) - do: + let t = getTypeDesc(p.module, ty) + p.module.s[cfsStrData].addVarWithInitializer(kind = Const, name = tmpName, typ = t): var closureInit: StructInitializer - data.addStructInitializer(closureInit, kind = siOrderedStruct): - data.addField(closureInit, name = "ClP_0"): - data.add("NIM_NIL") - data.addField(closureInit, name = "ClE_0"): - data.add("NIM_NIL") - p.module.s[cfsStrData].add(data) + p.module.s[cfsStrData].addStructInitializer(closureInit, kind = siOrderedStruct): + p.module.s[cfsStrData].addField(closureInit, name = "ClP_0"): + p.module.s[cfsStrData].add("NIM_NIL") + p.module.s[cfsStrData].addField(closureInit, name = "ClE_0"): + p.module.s[cfsStrData].add("NIM_NIL") result.add tmpName elif k in {tyPointer, tyNil, tyProc}: result.add rope("NIM_NIL") @@ -107,7 +104,7 @@ proc genRawSetData(cs: TBitSet, size: int; result: var Builder) = result.add "0123456789abcdef"[cs[i] div 16] result.add "0123456789abcdef"[cs[i] mod 16] else: - intLiteral(cast[BiggestInt](bitSetToWord(cs, size)), result) + result.addIntLiteral(cast[BiggestInt](bitSetToWord(cs, size))) proc genSetNode(p: BProc, n: PNode; result: var Builder) = var size = int(getSize(p.config, n.typ)) @@ -118,12 +115,9 @@ proc genSetNode(p: BProc, n: PNode; result: var Builder) = if id == p.module.labels: # not found in cache: inc(p.module.labels) - var data = newBuilder("") - data.addVarWithTypeAndInitializer(kind = Const, name = tmpName): - data.add(getTypeDesc(p.module, n.typ)) - do: - genRawSetData(cs, size, data) - p.module.s[cfsStrData].add(data) + let td = getTypeDesc(p.module, n.typ) + p.module.s[cfsStrData].addVarWithInitializer(kind = Const, name = tmpName, typ = td): + genRawSetData(cs, size, p.module.s[cfsStrData]) result.add tmpName else: genRawSetData(cs, size, result) @@ -1065,8 +1059,9 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = # passing around `TLineInfo` + the set of files in the project. msg.add toFileLineCol(p.config, e.info) & " " msg.add genFieldDefect(p.config, field.name.s, disc.sym) - var strLit = newRopeAppender() - genStringLiteral(p.module, newStrNode(nkStrLit, msg), strLit) + var strLitBuilder = newBuilder("") + genStringLiteral(p.module, newStrNode(nkStrLit, msg), strLitBuilder) + let strLit = extract(strLitBuilder) ## discriminant check let rt = rdLoc(test) @@ -1456,7 +1451,7 @@ proc genStrConcat(p: BProc, e: PNode, d: var TLoc) = var a: TLoc var tmp: TLoc = getTemp(p, e.typ) var L = 0 - var appends = newBuilder("") + var appends: seq[Snippet] = @[] var lens: seq[Snippet] = @[] for i in 0.. @@ -1553,13 +1542,11 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) = var b = initLocExpr(p, e[2]) let seqType = skipTypes(e[1].typ, {tyVar}) var call = initLoc(locCall, e, OnHeap) - var callRes = newBuilder("") let ra = rdLoc(a) - callRes.addCast(getTypeDesc(p.module, e[1].typ)): - callRes.addCall(cgsymValue(p.module, "incrSeqV3"), + call.snippet = cCast(getTypeDesc(p.module, e[1].typ), + cCall(cgsymValue(p.module, "incrSeqV3"), if not p.module.compileToCpp: cCast(ptrType("TGenericSeq"), ra) else: ra, - genTypeInfoV1(p.module, seqType, e.info)) - call.snippet = callRes + genTypeInfoV1(p.module, seqType, e.info))) # emit the write barrier if required, but we can always move here, so # use 'genRefAssign' for the seq. genRefAssign(p, a, call) @@ -1589,12 +1576,10 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) = if optTinyRtti in p.config.globalOptions: let fnName = cgsymValue(p.module, if needsInit: "nimNewObj" else: "nimNewObjUninit") - var bres = newBuilder("") - bres.addCast(getTypeDesc(p.module, typ)): - bres.addCall(fnName, + b.snippet = cCast(getTypeDesc(p.module, typ), + cCall(fnName, sizeExpr, - cAlignof(getTypeDesc(p.module, bt))) - b.snippet = bres + cAlignof(getTypeDesc(p.module, bt)))) genAssignment(p, a, b, {}) else: let ti = genTypeInfoV1(p.module, typ, a.lode.info) @@ -1625,12 +1610,10 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) = if p.config.selectedGC == gcGo: # newObjRC1() would clash with unsureAsgnRef() - which is used by gcGo to # implement the write barrier - var bres = newBuilder("") - bres.addCast(getTypeDesc(p.module, typ)): - bres.addCall(cgsymValue(p.module, "newObj"), + b.snippet = cCast(getTypeDesc(p.module, typ), + cCall(cgsymValue(p.module, "newObj"), ti, - sizeExpr) - b.snippet = bres + sizeExpr)) let raa = addrLoc(p.config, a) let rb = b.rdLoc p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "unsureAsgnRef"), @@ -1638,22 +1621,18 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) = rb) else: # use newObjRC1 as an optimization - var bres = newBuilder("") - bres.addCast(getTypeDesc(p.module, typ)): - bres.addCall(cgsymValue(p.module, "newObjRC1"), + b.snippet = cCast(getTypeDesc(p.module, typ), + cCall(cgsymValue(p.module, "newObjRC1"), ti, - sizeExpr) - b.snippet = bres + sizeExpr)) let ra = a.rdLoc let rb = b.rdLoc p.s(cpsStmts).addAssignment(ra, rb) else: - var bres = newBuilder("") - bres.addCast(getTypeDesc(p.module, typ)): - bres.addCall(cgsymValue(p.module, "newObj"), + b.snippet = cCast(getTypeDesc(p.module, typ), + cCall(cgsymValue(p.module, "newObj"), ti, - sizeExpr) - b.snippet = bres + sizeExpr)) genAssignment(p, a, b, {}) # set the object type: genObjectInit(p, cpsStmts, bt, a, constructRefObj) @@ -1686,20 +1665,16 @@ proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope; lenIsZero: bool) = let typinfo = genTypeInfoV1(p.module, seqtype, dest.lode.info) if p.config.selectedGC == gcGo: # we need the write barrier - var callRes = newBuilder("") - callRes.addCast(st): - callRes.addCall(cgsymValue(p.module, "newSeq"), typinfo, length) - call.snippet = callRes + call.snippet = cCast(st, + cCall(cgsymValue(p.module, "newSeq"), typinfo, length)) let rad = addrLoc(p.config, dest) let rc = call.rdLoc p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "unsureAsgnRef"), cCast("void**", rad), rc) else: - var callRes = newBuilder("") - callRes.addCast(st): - callRes.addCall(cgsymValue(p.module, "newSeqRC1"), typinfo, length) - call.snippet = callRes + call.snippet = cCast(st, + cCall(cgsymValue(p.module, "newSeqRC1"), typinfo, length)) let rd = dest.rdLoc let rc = call.rdLoc p.s(cpsStmts).addAssignment(rd, rc) @@ -1709,10 +1684,8 @@ proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope; lenIsZero: bool) = else: let st = getTypeDesc(p.module, seqtype) let typinfo = genTypeInfoV1(p.module, seqtype, dest.lode.info) - var callRes = newBuilder("") - callRes.addCast(st): - callRes.addCall(cgsymValue(p.module, "newSeq"), typinfo, length) - call.snippet = callRes + call.snippet = cCast(st, + cCall(cgsymValue(p.module, "newSeq"), typinfo, length)) genAssignment(p, dest, call, {}) proc genNewSeq(p: BProc, e: PNode) = @@ -1755,11 +1728,10 @@ proc genNewSeqOfCap(p: BProc; e: PNode; d: var TLoc) = else: if d.k == locNone: d = getTemp(p, e.typ, needsInit=false) # bug #22560 let ra = a.rdLoc - var dres = newBuilder("") - dres.addCast(getTypeDesc(p.module, seqtype)): - dres.addCall(cgsymValue(p.module, "nimNewSeqOfCap"), + let dres = cCast(getTypeDesc(p.module, seqtype), + cCall(cgsymValue(p.module, "nimNewSeqOfCap"), genTypeInfoV1(p.module, seqtype, e.info), - ra) + ra)) putIntoDest(p, d, e, dres) gcUsage(p.config, e) @@ -1771,15 +1743,14 @@ proc rawConstExpr(p: BProc, n: PNode; d: var TLoc) = if id == p.module.labels: # expression not found in the cache: inc(p.module.labels) + let td = getTypeDesc(p.module, t) var data = newBuilder("") - data.addVarWithTypeAndInitializer(kind = Const, name = d.snippet): - data.add(getTypeDesc(p.module, t)) - do: + data.addVarWithInitializer(kind = Const, name = d.snippet, typ = td): # bug #23627; when generating const object fields, it's likely that # we need to generate type infos for the object, which may be an object with # custom hooks. We need to generate potential consts in the hooks first. genBracedInit(p, n, isConst = true, t, data) - p.module.s[cfsData].add data + p.module.s[cfsData].add(extract(data)) proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool = if d.k == locNone and n.len > ord(n.kind == nkObjConstr) and n.isDeepConstExpr: @@ -1942,8 +1913,7 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) = if L < 10: for i in 0.. ($5)($3)){ #raiseRangeErrorNoArgs(); ", - [rdCharLoc(a), first, last, + [rdCharLoc(a), extract(first), extract(last), raiser, getTypeDesc(p.module, n0t)]) raiseInstr(p, p.s(cpsStmts)) linefmt p, cpsStmts, "}$n", [] @@ -2541,12 +2511,12 @@ proc genRangeChck(p: BProc, n: PNode, d: var TLoc) = "(NI64)" else: "" - var first = newRopeAppender() + var first = newBuilder("") genLiteral(p, n[1], dest, first) - var last = newRopeAppender() + var last = newBuilder("") genLiteral(p, n[2], dest, last) linefmt(p, cpsStmts, "if ($5($1) < $2 || $5($1) > $3){ $4($1, $2, $3); ", - [rdCharLoc(a), first, last, + [rdCharLoc(a), extract(first), extract(last), raiser, boundaryCast]) raiseInstr(p, p.s(cpsStmts)) linefmt p, cpsStmts, "}$n", [] @@ -2952,9 +2922,9 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) = a, b: TLoc var idx: TLoc if nfAllConst in e.flags: - var elem = newRopeAppender() + var elem = newBuilder("") genSetNode(p, e, elem) - putIntoDest(p, d, e, elem) + putIntoDest(p, d, e, extract(elem)) else: if d.k == locNone: d = getTemp(p, e.typ) if getSize(p.config, e.typ) > 8: @@ -3043,12 +3013,11 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) = if isConstClosure(n): inc(p.module.labels) var tmp = "CNSTCLOSURE" & rope(p.module.labels) + let td = getTypeDesc(p.module, n.typ) var data = newBuilder("") - data.addVarWithTypeAndInitializer(kind = Const, name = tmp): - data.add(getTypeDesc(p.module, n.typ)) - do: + data.addVarWithInitializer(kind = Const, name = tmp, typ = td): genBracedInit(p, n, isConst = true, n.typ, data) - p.module.s[cfsData].add data + p.module.s[cfsData].add(extract(data)) putIntoDest(p, d, n, tmp, OnStatic) else: var tmp: TLoc @@ -3075,8 +3044,7 @@ proc genArrayConstr(p: BProc, n: PNode, d: var TLoc) = if d.k == locNone: d = getTemp(p, n.typ) for i in 0.. 0: result.addf("FR_.len+=$1;$n", [b.frameLen.rope]) - result.add(b.sections[cpsInit]) - result.add(b.sections[cpsStmts]) + result.add(extract(b.sections[cpsInit])) + result.add(extract(b.sections[cpsStmts])) proc endBlock(p: BProc, blockEnd: Rope) = let topBlock = p.blocks.len-1 @@ -281,7 +281,7 @@ proc genGotoVar(p: BProc; value: PNode) = proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; result: var Builder) -proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Rope) = +proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Builder) = if lfDynamicLib in v.loc.flags or sfThread in v.flags or p.hcrOn: discard "nothing to do" elif sfGlobal in v.flags and value != nil and isDeepConstExpr(value, p.module.compileToCpp) and @@ -330,8 +330,9 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) = value.kind in nkCallKinds and value[0].kind == nkSym and v.typ.kind != tyPtr and sfConstructor in value[0].sym.flags var targetProc = p - var valueAsRope = "" - potentialValueInit(p, v, value, valueAsRope) + var valueBuilder = newBuilder("") + potentialValueInit(p, v, value, valueBuilder) + let valueAsRope = extract(valueBuilder) if sfGlobal in v.flags: if v.flags * {sfImportc, sfExportc} == {sfImportc} and value.kind == nkEmpty and @@ -594,8 +595,7 @@ proc genComputedGoto(p: BProc; n: PNode) = return let val = getOrdValue(it[j]) - var lit = newRopeAppender() - intLiteral(toInt64(val)+id+1, lit) + let lit = cIntLiteral(toInt64(val)+id+1) lineF(p, cpsStmts, "TMP$#_:$n", [lit]) genStmts(p, it.lastSon) @@ -775,7 +775,7 @@ proc finallyActions(p: BProc) = if finallyBlock != nil: genSimpleBlock(p, finallyBlock[0]) -proc raiseInstr(p: BProc; result: var Rope) = +proc raiseInstr(p: BProc; result: var Builder) = if p.config.exc == excGoto: let L = p.nestedTryStmts.len if L == 0: @@ -925,8 +925,7 @@ proc genStringCase(p: BProc, t: PNode, stringKind: TTypeKind, d: var TLoc) = [rdLoc(a), bitMask]) for j in 0..high(branches): if branches[j] != "": - var lit = newRopeAppender() - intLiteral(j, lit) + let lit = cIntLiteral(j) lineF(p, cpsStmts, "case $1: $n$2break;$n", [lit, branches[j]]) lineF(p, cpsStmts, "}$n", []) # else statement: @@ -964,22 +963,22 @@ proc genCaseRange(p: BProc, branch: PNode) = for j in 0..= 0 if optLineDir in conf.options and line > 0: if fileIdx == InvalidFileIdx: @@ -283,7 +294,7 @@ proc genCLineDir(r: var Rope, fileIdx: FileIndex, line: int; conf: ConfigRef) = else: r.add(rope("\n#line " & $line & " FX_" & $fileIdx.int32 & "\n")) -proc genCLineDir(r: var Rope, fileIdx: FileIndex, line: int; p: BProc; info: TLineInfo; lastFileIndex: FileIndex) = +proc genCLineDir(r: var Builder, fileIdx: FileIndex, line: int; p: BProc; info: TLineInfo; lastFileIndex: FileIndex) = assert line >= 0 if optLineDir in p.config.options and line > 0: if fileIdx == InvalidFileIdx: @@ -291,7 +302,7 @@ proc genCLineDir(r: var Rope, fileIdx: FileIndex, line: int; p: BProc; info: TLi else: r.add(rope("\n#line " & $line & " FX_" & $fileIdx.int32 & "\n")) -proc genCLineDir(r: var Rope, info: TLineInfo; conf: ConfigRef) = +proc genCLineDir(r: var Builder, info: TLineInfo; conf: ConfigRef) = if optLineDir in conf.options: genCLineDir(r, info.fileIndex, info.safeLineNm, conf) @@ -304,7 +315,7 @@ proc freshLineInfo(p: BProc; info: TLineInfo): bool = else: result = false -proc genCLineDir(r: var Rope, p: BProc, info: TLineInfo; conf: ConfigRef) = +proc genCLineDir(r: var Builder, p: BProc, info: TLineInfo; conf: ConfigRef) = if optLineDir in conf.options: let lastFileIndex = p.lastLineInfo.fileIndex if freshLineInfo(p, info): @@ -328,7 +339,7 @@ proc genLineDir(p: BProc, t: PNode) = proc accessThreadLocalVar(p: BProc, s: PSym) proc emulatedThreadVars(conf: ConfigRef): bool {.inline.} proc genProc(m: BModule, prc: PSym) -proc raiseInstr(p: BProc; result: var Rope) +proc raiseInstr(p: BProc; result: var Builder) template compileToCpp(m: BModule): untyped = m.config.backend == backendCpp or sfCompileToCpp in m.module.flags @@ -340,7 +351,6 @@ proc getTempName(m: BModule): Rope = proc isNoReturn(m: BModule; s: PSym): bool {.inline.} = sfNoReturn in s.flags and m.config.exc != excGoto -include cbuilderbase include cbuilderexprs include cbuilderdecls include cbuilderstmts @@ -613,28 +623,29 @@ proc getIntTemp(p: BProc): TLoc = linefmt(p, cpsLocals, "NI $1;$n", [result.snippet]) proc localVarDecl(p: BProc; n: PNode): Rope = - result = "" + var res = newBuilder("") let s = n.sym if s.loc.k == locNone: fillLocalName(p, s) fillLoc(s.loc, locLocalVar, n, OnStack) if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy) if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0: - result.addf("NIM_ALIGN($1) ", [rope(s.alignment)]) + res.addf("NIM_ALIGN($1) ", [rope(s.alignment)]) - genCLineDir(result, p, n.info, p.config) + genCLineDir(res, p, n.info, p.config) - result.add getTypeDesc(p.module, s.typ, dkVar) + res.add getTypeDesc(p.module, s.typ, dkVar) if sfCodegenDecl notin s.flags: - if sfRegister in s.flags: result.add(" register") + if sfRegister in s.flags: res.add(" register") #elif skipTypes(s.typ, abstractInst).kind in GcTypeKinds: # decl.add(" GC_GUARD") - if sfVolatile in s.flags: result.add(" volatile") - if sfNoalias in s.flags: result.add(" NIM_NOALIAS") - result.add(" ") - result.add(s.loc.snippet) + if sfVolatile in s.flags: res.add(" volatile") + if sfNoalias in s.flags: res.add(" NIM_NOALIAS") + res.add(" ") + res.add(s.loc.snippet) + result = extract(res) else: - result = runtimeFormat(s.cgDeclFrmt, [result, s.loc.snippet]) + result = runtimeFormat(s.cgDeclFrmt, [extract(res), s.loc.snippet]) proc assignLocalVar(p: BProc, n: PNode) = #assert(s.loc.k == locNone) # not yet assigned @@ -770,7 +781,7 @@ proc genStmts(p: BProc, t: PNode) proc expr(p: BProc, n: PNode, d: var TLoc) proc putLocIntoDest(p: BProc, d: var TLoc, s: TLoc) -proc genLiteral(p: BProc, n: PNode; result: var Rope) +proc genLiteral(p: BProc, n: PNode; result: var Builder) proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope; argsCounter: var int) proc raiseExit(p: BProc) proc raiseExitCleanup(p: BProc, destroy: string) @@ -807,7 +818,7 @@ $1define nimlf_(n, file) \ FR_.line = n; FR_.filename = file; """ - if p.module.s[cfsFrameDefines].len == 0: + if p.module.s[cfsFrameDefines].buf.len == 0: appcg(p.module, p.module.s[cfsFrameDefines], frameDefines, ["#"]) cgsym(p.module, "nimFrame") @@ -848,7 +859,7 @@ proc loadDynamicLib(m: BModule, lib: PLib) = var s: TStringSeq = @[] libCandidates(lib.path.strVal, s) rawMessage(m.config, hintDependency, lib.path.strVal) - var loadlib: Rope = "" + var loadlib = newBuilder("") for i in 0..high(s): inc(m.labels) if i > 0: loadlib.add("||") @@ -859,7 +870,7 @@ proc loadDynamicLib(m: BModule, lib: PLib) = loadlib.addf "))$n", [] appcg(m, m.s[cfsDynLibInit], "if (!($1)) #nimLoadLibraryError(", - [loadlib]) + [extract(loadlib)]) genStringLiteral(m, lib.path, m.s[cfsDynLibInit]) m.s[cfsDynLibInit].addf ");$n", [] @@ -873,9 +884,9 @@ proc loadDynamicLib(m: BModule, lib: PLib) = [getTypeDesc(m, lib.path.typ, dkVar), rdLoc(dest)]) expr(p, lib.path, dest) - m.s[cfsVars].add(p.s(cpsLocals)) - m.s[cfsDynLibInit].add(p.s(cpsInit)) - m.s[cfsDynLibInit].add(p.s(cpsStmts)) + m.s[cfsVars].add(extract(p.s(cpsLocals))) + m.s[cfsDynLibInit].add(extract(p.s(cpsInit))) + m.s[cfsDynLibInit].add(extract(p.s(cpsStmts))) appcg(m, m.s[cfsDynLibInit], "if (!($1 = #nimLoadLibrary($2))) #nimLoadLibraryError($2);$n", [tmp, rdLoc(dest)]) @@ -995,12 +1006,12 @@ proc generateHeaders(m: BModule) = #undef unix """) -proc openNamespaceNim(namespace: string; result: var Rope) = +proc openNamespaceNim(namespace: string; result: var Builder) = result.add("namespace ") result.add(namespace) result.add(" {\L") -proc closeNamespaceNim(result: var Rope) = +proc closeNamespaceNim(result: var Builder) = result.add("}\L") proc closureSetup(p: BProc, prc: PSym) = @@ -1185,9 +1196,10 @@ proc getProcTypeCast(m: BModule, prc: PSym): Rope = result = getTypeDesc(m, prc.loc.t) if prc.typ.callConv == ccClosure: var rettype: Snippet = "" - var params = newBuilder("") + var desc = newBuilder("") var check = initIntSet() - genProcParams(m, prc.typ, rettype, params, check) + genProcParams(m, prc.typ, rettype, desc, check) + let params = extract(desc) result = procPtrTypeUnnamed(rettype = rettype, params = params) proc genProcBody(p: BProc; procBody: PNode) = @@ -1201,14 +1213,14 @@ proc genProcBody(p: BProc; procBody: PNode) = proc genProcAux*(m: BModule, prc: PSym) = var p = newProc(prc, m) - var header = newRopeAppender() + var header = newBuilder("") let isCppMember = m.config.backend == backendCpp and sfCppMember * prc.flags != {} var visibility: DeclVisibility = None if isCppMember: genMemberProcHeader(m, prc, header) else: genProcHeader(m, prc, header, visibility, asPtr = false, addAttributes = false) - var returnStmt: Rope = "" + var returnStmt: Snippet = "" assert(prc.ast != nil) var procBody = transformBody(m.g.graph, m.idgen, prc, {}) @@ -1241,9 +1253,10 @@ proc genProcAux*(m: BModule, prc: PSym) = discard "result init optimized out" else: initLocalVar(p, res, immediateAsgn=false) - returnStmt = "\t" + var returnBuilder = newBuilder("\t") let rres = rdLoc(res.loc) - returnStmt.addReturn(rres) + returnBuilder.addReturn(rres) + returnStmt = extract(returnBuilder) elif sfConstructor in prc.flags: resNode.sym.loc.flags.incl lfIndirect fillLoc(resNode.sym.loc, locParam, resNode, "this", OnHeap) @@ -1278,27 +1291,27 @@ proc genProcAux*(m: BModule, prc: PSym) = generatedProc.genCLineDir prc.info, m.config generatedProc.addDeclWithVisibility(visibility): if sfPure in prc.flags: - generatedProc.add(header) + generatedProc.add(extract(header)) generatedProc.finishProcHeaderWithBody(): - generatedProc.add(p.s(cpsLocals)) - generatedProc.add(p.s(cpsInit)) - generatedProc.add(p.s(cpsStmts)) + generatedProc.add(extract(p.s(cpsLocals))) + generatedProc.add(extract(p.s(cpsInit))) + generatedProc.add(extract(p.s(cpsStmts))) else: if m.hcrOn and isReloadable(m, prc): m.s[cfsProcHeaders].addDeclWithVisibility(visibility): # Add forward declaration for "_actual"-suffixed functions defined in the same module (or inline). # This fixes the use of methods and also the case when 2 functions within the same module # call each other using directly the "_actual" versions (an optimization) - see issue #11608 - m.s[cfsProcHeaders].add(header) + m.s[cfsProcHeaders].add(extract(header)) m.s[cfsProcHeaders].finishProcHeaderAsProto() - generatedProc.add(header) + generatedProc.add(extract(header)) generatedProc.finishProcHeaderWithBody(): if optStackTrace in prc.options: - generatedProc.add(p.s(cpsLocals)) + generatedProc.add(extract(p.s(cpsLocals))) var procname = makeCString(prc.name.s) generatedProc.add(initFrame(p, procname, quotedFilename(p.config, prc.info))) else: - generatedProc.add(p.s(cpsLocals)) + generatedProc.add(extract(p.s(cpsLocals))) if optProfiler in prc.options: # invoke at proc entry for recursion: p.s(cpsInit).add('\t') @@ -1307,15 +1320,15 @@ proc genProcAux*(m: BModule, prc: PSym) = # this pair of {} is required for C++ (C++ is weird with its # control flow integrity checks): generatedProc.addScope(): - generatedProc.add(p.s(cpsInit)) - generatedProc.add(p.s(cpsStmts)) + generatedProc.add(extract(p.s(cpsInit))) + generatedProc.add(extract(p.s(cpsStmts))) generatedProc.addLabel("BeforeRet_") else: - generatedProc.add(p.s(cpsInit)) - generatedProc.add(p.s(cpsStmts)) + generatedProc.add(extract(p.s(cpsInit))) + generatedProc.add(extract(p.s(cpsStmts))) if optStackTrace in prc.options: generatedProc.add(deinitFrame(p)) generatedProc.add(returnStmt) - m.s[cfsProcs].add(generatedProc) + m.s[cfsProcs].add(extract(generatedProc)) if isReloadable(m, prc): m.s[cfsDynLibInit].add('\t') m.s[cfsDynLibInit].addAssignmentWithValue(prc.loc.snippet): @@ -1356,13 +1369,13 @@ proc genProcPrototype(m: BModule, sym: PSym) = '"' & name & '"') elif not containsOrIncl(m.declaredProtos, sym.id): let asPtr = isReloadable(m, sym) - var header = newRopeAppender() + var header = newBuilder("") var visibility: DeclVisibility = None genProcHeader(m, sym, header, visibility, asPtr = asPtr, addAttributes = true) if asPtr: m.s[cfsProcHeaders].addDeclWithVisibility(visibility): # genProcHeader would give variable declaration, add it directly - m.s[cfsProcHeaders].add(header) + m.s[cfsProcHeaders].add(extract(header)) else: let extraVis = if sym.typ.callConv != ccInline and requiresExternC(m, sym): @@ -1371,7 +1384,7 @@ proc genProcPrototype(m: BModule, sym: PSym) = None m.s[cfsProcHeaders].addDeclWithVisibility(extraVis): m.s[cfsProcHeaders].addDeclWithVisibility(visibility): - m.s[cfsProcHeaders].add(header) + m.s[cfsProcHeaders].add(extract(header)) m.s[cfsProcHeaders].finishProcHeaderAsProto() # TODO: figure out how to rename this - it DOES generate a forward declaration @@ -1498,7 +1511,7 @@ proc genVarPrototype(m: BModule, n: PNode) = "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.snippet, getTypeDesc(m, sym.loc.t, dkVar), getModuleDllPath(m, sym)]) -proc addNimDefines(result: var Rope; conf: ConfigRef) {.inline.} = +proc addNimDefines(result: var Builder; conf: ConfigRef) {.inline.} = result.addf("#define NIM_INTBITS $1\L", [ platform.CPU[conf.target.targetCPU].intSize.rope]) if conf.cppCustomNamespace.len > 0: @@ -1522,9 +1535,10 @@ proc getCopyright(conf: ConfigRef; cfile: Cfile): Rope = rope(getCompileCFileCmd(conf, cfile))] proc getFileHeader(conf: ConfigRef; cfile: Cfile): Rope = - result = getCopyright(conf, cfile) - if conf.hcrOn: result.add("#define NIM_HOT_CODE_RELOADING\L") - addNimDefines(result, conf) + var res = newBuilder(getCopyright(conf, cfile)) + if conf.hcrOn: res.add("#define NIM_HOT_CODE_RELOADING\L") + addNimDefines(res, conf) + result = extract(res) proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope = ## Returns a mangled module name. @@ -1567,8 +1581,9 @@ proc genMainProc(m: BModule) = assert prc != nil let n = newStrNode(nkStrLit, prc.annex.path.strVal) n.info = prc.annex.path.info - var strLit = newRopeAppender() - genStringLiteral(m, n, strLit) + var strLitBuilder = newBuilder("") + genStringLiteral(m, n, strLitBuilder) + let strLit = extract(strLitBuilder) appcg(m, result, "\tif (!($1 = #nimLoadLibrary($2)))$N" & "\t\t#nimLoadLibraryError($2);$N", [handle, strLit]) @@ -1780,10 +1795,10 @@ proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) proc whichInitProcs*(m: BModule): set[ModuleBackendFlag] = # called from IC. result = {} - if m.hcrOn or m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: + if m.hcrOn or m.preInitProc.s(cpsInit).buf.len > 0 or m.preInitProc.s(cpsStmts).buf.len > 0: result.incl HasModuleInitProc for i in cfsTypeInit1..cfsDynLibInit: - if m.s[i].len != 0: + if m.s[i].buf.len != 0: result.incl HasDatInitProc break @@ -1836,7 +1851,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = m.s[cfsInitProc].add(hcrModuleMeta) return - if m.s[cfsDatInitProc].len > 0: + if m.s[cfsDatInitProc].buf.len > 0: g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) g.mainDatInit.addf("\t$1();$N", [datInit]) @@ -1848,7 +1863,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcAtomicArc, gcOrc}: g.mainDatInit.add(ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", [])) - if m.s[cfsInitProc].len > 0: + if m.s[cfsInitProc].buf.len > 0: g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) let initCall = "\t$1();$N" % [init] if sfMainModule in m.module.flags: @@ -1865,22 +1880,22 @@ proc genDatInitCode(m: BModule) = var moduleDatInitRequired = m.hcrOn - var prc = "$1 N_NIMCALL(void, $2)(void) {$N" % - [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), getDatInitName(m)] + var prc = newBuilder("$1 N_NIMCALL(void, $2)(void) {$N" % + [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), getDatInitName(m)]) # we don't want to break into such init code - could happen if a line # directive from a function written by the user spills after itself genCLineDir(prc, InvalidFileIdx, 999999, m.config) for i in cfsTypeInit1..cfsDynLibInit: - if m.s[i].len != 0: + if m.s[i].buf.len != 0: moduleDatInitRequired = true - prc.add(m.s[i]) + prc.add(extract(m.s[i])) prc.addf("}$N$N", []) if moduleDatInitRequired: - m.s[cfsDatInitProc].add(prc) + m.s[cfsDatInitProc].add(extract(prc)) #rememberFlag(m.g.graph, m.module, HasDatInitProc) # Very similar to the contents of symInDynamicLib - basically only the @@ -1907,8 +1922,8 @@ proc genInitCode(m: BModule) = ## into other modules, only simple rope manipulations are allowed var moduleInitRequired = m.hcrOn let initname = getInitName(m) - var prc = "$1 N_NIMCALL(void, $2)(void) {$N" % - [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), initname] + var prc = newBuilder("$1 N_NIMCALL(void, $2)(void) {$N" % + [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), initname]) # we don't want to break into such init code - could happen if a line # directive from a function written by the user spills after itself genCLineDir(prc, InvalidFileIdx, 999999, m.config) @@ -1931,13 +1946,13 @@ proc genInitCode(m: BModule) = [getModuleDllPath(m, m.module)]) template writeSection(thing: untyped, section: TCProcSection, addHcrGuards = false) = - if m.thing.s(section).len > 0: + if m.thing.s(section).buf.len > 0: moduleInitRequired = true if addHcrGuards: prc.add("\tif (nim_hcr_do_init_) {\n\n") - prc.add(m.thing.s(section)) + prc.add(extract(m.thing.s(section))) if addHcrGuards: prc.add("\n\t} // nim_hcr_do_init_\n") - if m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: + if m.preInitProc.s(cpsInit).buf.len > 0 or m.preInitProc.s(cpsStmts).buf.len > 0: # Give this small function its own scope prc.addf("{$N", []) # Keep a bogus frame in case the code needs one @@ -1957,7 +1972,7 @@ proc genInitCode(m: BModule) = prc.addf("{$N", []) writeSection(initProc, cpsLocals) - if m.initProc.s(cpsInit).len > 0 or m.initProc.s(cpsStmts).len > 0: + if m.initProc.s(cpsInit).buf.len > 0 or m.initProc.s(cpsStmts).buf.len > 0: moduleInitRequired = true if optStackTrace in m.initProc.options and frameDeclared notin m.flags: # BUT: the generated init code might depend on a current frame, so @@ -2011,14 +2026,14 @@ proc genInitCode(m: BModule) = prc.add(ex) if moduleInitRequired or sfMainModule in m.module.flags: - m.s[cfsInitProc].add(prc) + m.s[cfsInitProc].add(extract(prc)) #rememberFlag(m.g.graph, m.module, HasModuleInitProc) genDatInitCode(m) if m.hcrOn: m.s[cfsInitProc].addf("N_LIB_EXPORT N_NIMCALL(void, HcrCreateTypeInfos)(void) {$N", []) - m.s[cfsInitProc].add(m.hcrCreateTypeInfosProc) + m.s[cfsInitProc].add(extract(m.hcrCreateTypeInfosProc)) m.s[cfsInitProc].addf("}$N$N", []) registerModuleToMain(m.g, m) @@ -2060,31 +2075,32 @@ proc postprocessCode(conf: ConfigRef, r: var Rope) = proc genModule(m: BModule, cfile: Cfile): Rope = var moduleIsEmpty = true - result = getFileHeader(m.config, cfile) + var res = newBuilder(getFileHeader(m.config, cfile)) generateThreadLocalStorage(m) generateHeaders(m) - result.add(m.s[cfsHeaders]) + res.add(extract(m.s[cfsHeaders])) if m.config.cppCustomNamespace.len > 0: - openNamespaceNim(m.config.cppCustomNamespace, result) - if m.s[cfsFrameDefines].len > 0: - result.add(m.s[cfsFrameDefines]) + openNamespaceNim(m.config.cppCustomNamespace, res) + if m.s[cfsFrameDefines].buf.len > 0: + res.add(extract(m.s[cfsFrameDefines])) for i in cfsForwardTypes..cfsProcs: - if m.s[i].len > 0: + if m.s[i].buf.len > 0: moduleIsEmpty = false - result.add(m.s[i]) + res.add(extract(m.s[i])) - if m.s[cfsInitProc].len > 0: + if m.s[cfsInitProc].buf.len > 0: moduleIsEmpty = false - result.add(m.s[cfsInitProc]) - if m.s[cfsDatInitProc].len > 0 or m.hcrOn: + res.add(extract(m.s[cfsInitProc])) + if m.s[cfsDatInitProc].buf.len > 0 or m.hcrOn: moduleIsEmpty = false - result.add(m.s[cfsDatInitProc]) + res.add(extract(m.s[cfsDatInitProc])) if m.config.cppCustomNamespace.len > 0: - closeNamespaceNim(result) + closeNamespaceNim(res) + result = extract(res) if optLineDir in m.config.options: var srcFileDefs = "" for fi in 0..m.config.m.fileInfos.high: @@ -2115,11 +2131,12 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule result.typeInfoMarker = initTable[SigHash, Rope]() result.sigConflicts = initCountTable[SigHash]() result.initProc = newProc(nil, result) - for i in low(result.s)..high(result.s): result.s[i] = newRopeAppender() + for i in low(result.s)..high(result.s): result.s[i] = newBuilder("") result.initProc.options = initProcOptions(result) result.preInitProc = newProc(nil, result) result.preInitProc.flags.incl nimErrorFlagDisabled result.preInitProc.labels = 100_000 # little hack so that unique temporaries are generated + result.hcrCreateTypeInfosProc = newBuilder("") result.dataCache = initNodeTable() result.typeStack = @[] result.typeNodesName = getTempName(result) @@ -2158,7 +2175,7 @@ proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassCont incl g.generatedHeader.flags, isHeaderFile proc writeHeader(m: BModule) = - var result = headerTop() + var result = newBuilder(headerTop()) var guard = "__$1__" % [m.filename.splitFile.name.rope] result.addf("#ifndef $1$n#define $1$n", [guard]) addNimDefines(result, m.config) @@ -2166,17 +2183,17 @@ proc writeHeader(m: BModule) = generateThreadLocalStorage(m) for i in cfsHeaders..cfsProcs: - result.add(m.s[i]) + result.add(extract(m.s[i])) if m.config.cppCustomNamespace.len > 0 and i == cfsHeaders: openNamespaceNim(m.config.cppCustomNamespace, result) - result.add(m.s[cfsInitProc]) + result.add(extract(m.s[cfsInitProc])) if optGenDynLib in m.config.globalOptions: result.add("N_LIB_IMPORT ") result.addf("N_CDECL(void, $1NimMain)(void);$n", [rope m.config.nimMainPrefix]) if m.config.cppCustomNamespace.len > 0: closeNamespaceNim(result) result.addf("#endif /* $1 */$n", [guard]) - if not writeRope(result, m.filename): + if not writeRope(extract(result), m.filename): rawMessage(m.config, errCannotOpenFile, m.filename.string) proc getCFile(m: BModule): AbsoluteFile = diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 7bc273df300d..3dc042174fd5 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -11,7 +11,7 @@ import ast, ropes, options, - lineinfos, pathutils, modulegraphs + lineinfos, pathutils, modulegraphs, cbuilderbase import std/[intsets, tables, sets] @@ -43,12 +43,12 @@ type ctUInt, ctUInt8, ctUInt16, ctUInt32, ctUInt64, ctArray, ctPtrToArray, ctStruct, ctPtr, ctNimStr, ctNimSeq, ctProc, ctCString - TCFileSections* = array[TCFileSection, Rope] # represents a generated C file + TCFileSections* = array[TCFileSection, Builder] # represents a generated C file TCProcSection* = enum # the sections a generated C proc consists of cpsLocals, # section of local variables for C proc cpsInit, # section for init of variables for C proc cpsStmts # section of local statements for C proc - TCProcSections* = array[TCProcSection, Rope] # represents a generated C proc + TCProcSections* = array[TCProcSection, Builder] # represents a generated C proc BModule* = ref TCGen BProc* = ref TCProc TBlock* = object @@ -161,7 +161,7 @@ type typeInfoMarkerV2*: TypeCache initProc*: BProc # code for init procedure preInitProc*: BProc # code executed before the init proc - hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on + hcrCreateTypeInfosProc*: Builder # type info globals are in here when HCR=on inHcrInitGuard*: bool # We are currently within a HCR reloading guard. typeStack*: TTypeSeq # used for type generation dataCache*: TNodeTable @@ -181,18 +181,18 @@ proc includeHeader*(this: BModule; header: string) = if not this.headerFiles.contains header: this.headerFiles.add header -proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} = +proc s*(p: BProc, s: TCProcSection): var Builder {.inline.} = # section in the current block result = p.blocks[^1].sections[s] -proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} = +proc procSec*(p: BProc, s: TCProcSection): var Builder {.inline.} = # top level proc sections result = p.blocks[0].sections[s] proc initBlock*(): TBlock = result = TBlock() for i in low(result.sections)..high(result.sections): - result.sections[i] = newRopeAppender() + result.sections[i] = newBuilder("") proc newProc*(prc: PSym, module: BModule): BProc = result = BProc( From f98964d99f2ab721cc718b2be3afa78305867b6b Mon Sep 17 00:00:00 2001 From: metagn Date: Sun, 3 Nov 2024 19:57:31 +0300 Subject: [PATCH 5/5] cbuilder: add `for` range statements (#24391) Finishes `genEnumInfo` as followup to #24351. As #24381 mentions this covers every use of `for` loops in the codegen. --------- Co-authored-by: Andreas Rumpf --- compiler/cbuilderstmts.nim | 26 ++++++++++++++++++++++++++ compiler/ccgtypes.nim | 22 +++++++++++++++------- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/compiler/cbuilderstmts.nim b/compiler/cbuilderstmts.nim index a96571d1b8b4..e2d272e79524 100644 --- a/compiler/cbuilderstmts.nim +++ b/compiler/cbuilderstmts.nim @@ -93,6 +93,32 @@ template addElseBranch(builder: var Builder, stmt: var IfStmt, body: typed) = body builder.add("}") +proc addForRangeHeader(builder: var Builder, i, start, bound: Snippet, inclusive: bool = false) = + builder.add("for (") + builder.add(i) + builder.add(" = ") + builder.add(start) + builder.add("; ") + builder.add(i) + if inclusive: + builder.add(" <= ") + else: + builder.add(" < ") + builder.add(bound) + builder.add("; ") + builder.add(i) + builder.add("++) {\n") + +template addForRangeExclusive(builder: var Builder, i, start, bound: Snippet, body: typed) = + addForRangeHeader(builder, i, start, bound, false) + body + builder.add("}\n") + +template addForRangeInclusive(builder: var Builder, i, start, bound: Snippet, body: typed) = + addForRangeHeader(builder, i, start, bound, true) + body + builder.add("}\n") + template addScope(builder: var Builder, body: typed) = builder.add("{") body diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index eb6b6bd9f31f..04b740aa92d9 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1503,13 +1503,20 @@ proc genEnumInfo(m: BModule; typ: PType, name: Rope; info: TLineInfo) = m.s[cfsTypeInit1].addArrayVarWithInitializer( kind = Global, name = enumArray, - elementType = "char* NIM_CONST", # XXX maybe do this in `addVar` + elementType = constPtrType("char"), len = typ.n.len): m.s[cfsTypeInit1].add(extract(enumNames)) - m.s[cfsTypeInit3].addf("for ($1 = 0; $1 < $2; $1++) {$n" & - "$3[$1+$4].kind = 1;$n" & "$3[$1+$4].offset = $1;$n" & - "$3[$1+$4].name = $5[$1];$n" & "$6[$1] = &$3[$1+$4];$n" & "}$n", [counter, - rope(typ.n.len), m.typeNodesName, rope(firstNimNode), enumArray, nodePtrs]) + m.s[cfsTypeInit3].addForRangeExclusive(i = counter, + start = cIntValue(0), + bound = cIntValue(typ.n.len)): + let nodeLoc = subscript(m.typeNodesName, + cOp(Add, "NI", counter, cIntValue(firstNimNode))) + m.s[cfsTypeInit3].addFieldAssignment(nodeLoc, "kind", cIntValue(1)) + m.s[cfsTypeInit3].addFieldAssignment(nodeLoc, "offset", counter) + m.s[cfsTypeInit3].addFieldAssignment(nodeLoc, "name", + subscript(enumArray, counter)) + m.s[cfsTypeInit3].addSubscriptAssignment(nodePtrs, counter, + cAddr(nodeLoc)) m.s[cfsTypeInit3].add(extract(specialCases)) let n = getNimNode(m) m.s[cfsTypeInit3].addFieldAssignment(n, "len", typ.n.len) @@ -1518,8 +1525,9 @@ proc genEnumInfo(m: BModule; typ: PType, name: Rope; info: TLineInfo) = cAddr(subscript(nodePtrs, cIntValue(0)))) m.s[cfsTypeInit3].addFieldAssignment(tiNameForHcr(m, name), "node", cAddr(n)) if hasHoles: - # 1 << 2 is {ntfEnumHole} - m.s[cfsTypeInit3].addf("$1.flags = 1<<2;$n", [tiNameForHcr(m, name)]) + m.s[cfsTypeInit3].addFieldAssignment(tiNameForHcr(m, name), "flags", + # 1 << 2 is {ntfEnumHole} + cOp(Shl, "NU8", cIntValue(1), cIntValue(2))) proc genSetInfo(m: BModule; typ: PType, name: Rope; info: TLineInfo) = assert(typ.elementType != nil)