pump.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856
  1. #!/usr/bin/env python
  2. #
  3. # Copyright 2008, Google Inc.
  4. # All rights reserved.
  5. #
  6. # Redistribution and use in source and binary forms, with or without
  7. # modification, are permitted provided that the following conditions are
  8. # met:
  9. #
  10. # * Redistributions of source code must retain the above copyright
  11. # notice, this list of conditions and the following disclaimer.
  12. # * Redistributions in binary form must reproduce the above
  13. # copyright notice, this list of conditions and the following disclaimer
  14. # in the documentation and/or other materials provided with the
  15. # distribution.
  16. # * Neither the name of Google Inc. nor the names of its
  17. # contributors may be used to endorse or promote products derived from
  18. # this software without specific prior written permission.
  19. #
  20. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31. """pump v0.2.0 - Pretty Useful for Meta Programming.
  32. A tool for preprocessor meta programming. Useful for generating
  33. repetitive boilerplate code. Especially useful for writing C++
  34. classes, functions, macros, and templates that need to work with
  35. various number of arguments.
  36. USAGE:
  37. pump.py SOURCE_FILE
  38. EXAMPLES:
  39. pump.py foo.cc.pump
  40. Converts foo.cc.pump to foo.cc.
  41. GRAMMAR:
  42. CODE ::= ATOMIC_CODE*
  43. ATOMIC_CODE ::= $var ID = EXPRESSION
  44. | $var ID = [[ CODE ]]
  45. | $range ID EXPRESSION..EXPRESSION
  46. | $for ID SEPARATOR [[ CODE ]]
  47. | $($)
  48. | $ID
  49. | $(EXPRESSION)
  50. | $if EXPRESSION [[ CODE ]] ELSE_BRANCH
  51. | [[ CODE ]]
  52. | RAW_CODE
  53. SEPARATOR ::= RAW_CODE | EMPTY
  54. ELSE_BRANCH ::= $else [[ CODE ]]
  55. | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH
  56. | EMPTY
  57. EXPRESSION has Python syntax.
  58. """
  59. from __future__ import print_function
  60. import io
  61. import os
  62. import re
  63. import sys
  64. TOKEN_TABLE = [
  65. (re.compile(r'\$var\s+'), '$var'),
  66. (re.compile(r'\$elif\s+'), '$elif'),
  67. (re.compile(r'\$else\s+'), '$else'),
  68. (re.compile(r'\$for\s+'), '$for'),
  69. (re.compile(r'\$if\s+'), '$if'),
  70. (re.compile(r'\$range\s+'), '$range'),
  71. (re.compile(r'\$[_A-Za-z]\w*'), '$id'),
  72. (re.compile(r'\$\(\$\)'), '$($)'),
  73. (re.compile(r'\$'), '$'),
  74. (re.compile(r'\[\[\n?'), '[['),
  75. (re.compile(r'\]\]\n?'), ']]'),
  76. ]
  77. class Cursor:
  78. """Represents a position (line and column) in a text file."""
  79. def __init__(self, line=-1, column=-1):
  80. self.line = line
  81. self.column = column
  82. def __eq__(self, rhs):
  83. return self.line == rhs.line and self.column == rhs.column
  84. def __ne__(self, rhs):
  85. return not self == rhs
  86. def __lt__(self, rhs):
  87. return self.line < rhs.line or (
  88. self.line == rhs.line and self.column < rhs.column)
  89. def __le__(self, rhs):
  90. return self < rhs or self == rhs
  91. def __gt__(self, rhs):
  92. return rhs < self
  93. def __ge__(self, rhs):
  94. return rhs <= self
  95. def __str__(self):
  96. if self == Eof():
  97. return 'EOF'
  98. else:
  99. return '%s(%s)' % (self.line + 1, self.column)
  100. def __add__(self, offset):
  101. return Cursor(self.line, self.column + offset)
  102. def __sub__(self, offset):
  103. return Cursor(self.line, self.column - offset)
  104. def Clone(self):
  105. """Returns a copy of self."""
  106. return Cursor(self.line, self.column)
  107. # Special cursor to indicate the end-of-file.
  108. def Eof():
  109. """Returns the special cursor to denote the end-of-file."""
  110. return Cursor(-1, -1)
  111. class Token:
  112. """Represents a token in a Pump source file."""
  113. def __init__(self, start=None, end=None, value=None, token_type=None):
  114. if start is None:
  115. self.start = Eof()
  116. else:
  117. self.start = start
  118. if end is None:
  119. self.end = Eof()
  120. else:
  121. self.end = end
  122. self.value = value
  123. self.token_type = token_type
  124. def __str__(self):
  125. return 'Token @%s: \'%s\' type=%s' % (
  126. self.start, self.value, self.token_type)
  127. def Clone(self):
  128. """Returns a copy of self."""
  129. return Token(self.start.Clone(), self.end.Clone(), self.value,
  130. self.token_type)
  131. def StartsWith(lines, pos, string):
  132. """Returns True iff the given position in lines starts with 'string'."""
  133. return lines[pos.line][pos.column:].startswith(string)
  134. def FindFirstInLine(line, token_table):
  135. best_match_start = -1
  136. for (regex, token_type) in token_table:
  137. m = regex.search(line)
  138. if m:
  139. # We found regex in lines
  140. if best_match_start < 0 or m.start() < best_match_start:
  141. best_match_start = m.start()
  142. best_match_length = m.end() - m.start()
  143. best_match_token_type = token_type
  144. if best_match_start < 0:
  145. return None
  146. return (best_match_start, best_match_length, best_match_token_type)
  147. def FindFirst(lines, token_table, cursor):
  148. """Finds the first occurrence of any string in strings in lines."""
  149. start = cursor.Clone()
  150. cur_line_number = cursor.line
  151. for line in lines[start.line:]:
  152. if cur_line_number == start.line:
  153. line = line[start.column:]
  154. m = FindFirstInLine(line, token_table)
  155. if m:
  156. # We found a regex in line.
  157. (start_column, length, token_type) = m
  158. if cur_line_number == start.line:
  159. start_column += start.column
  160. found_start = Cursor(cur_line_number, start_column)
  161. found_end = found_start + length
  162. return MakeToken(lines, found_start, found_end, token_type)
  163. cur_line_number += 1
  164. # We failed to find str in lines
  165. return None
  166. def SubString(lines, start, end):
  167. """Returns a substring in lines."""
  168. if end == Eof():
  169. end = Cursor(len(lines) - 1, len(lines[-1]))
  170. if start >= end:
  171. return ''
  172. if start.line == end.line:
  173. return lines[start.line][start.column:end.column]
  174. result_lines = ([lines[start.line][start.column:]] +
  175. lines[start.line + 1:end.line] +
  176. [lines[end.line][:end.column]])
  177. return ''.join(result_lines)
  178. def StripMetaComments(str):
  179. """Strip meta comments from each line in the given string."""
  180. # First, completely remove lines containing nothing but a meta
  181. # comment, including the trailing \n.
  182. str = re.sub(r'^\s*\$\$.*\n', '', str)
  183. # Then, remove meta comments from contentful lines.
  184. return re.sub(r'\s*\$\$.*', '', str)
  185. def MakeToken(lines, start, end, token_type):
  186. """Creates a new instance of Token."""
  187. return Token(start, end, SubString(lines, start, end), token_type)
  188. def ParseToken(lines, pos, regex, token_type):
  189. line = lines[pos.line][pos.column:]
  190. m = regex.search(line)
  191. if m and not m.start():
  192. return MakeToken(lines, pos, pos + m.end(), token_type)
  193. else:
  194. print('ERROR: %s expected at %s.' % (token_type, pos))
  195. sys.exit(1)
  196. ID_REGEX = re.compile(r'[_A-Za-z]\w*')
  197. EQ_REGEX = re.compile(r'=')
  198. REST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)')
  199. OPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*')
  200. WHITE_SPACE_REGEX = re.compile(r'\s')
  201. DOT_DOT_REGEX = re.compile(r'\.\.')
  202. def Skip(lines, pos, regex):
  203. line = lines[pos.line][pos.column:]
  204. m = re.search(regex, line)
  205. if m and not m.start():
  206. return pos + m.end()
  207. else:
  208. return pos
  209. def SkipUntil(lines, pos, regex, token_type):
  210. line = lines[pos.line][pos.column:]
  211. m = re.search(regex, line)
  212. if m:
  213. return pos + m.start()
  214. else:
  215. print ('ERROR: %s expected on line %s after column %s.' %
  216. (token_type, pos.line + 1, pos.column))
  217. sys.exit(1)
  218. def ParseExpTokenInParens(lines, pos):
  219. def ParseInParens(pos):
  220. pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
  221. pos = Skip(lines, pos, r'\(')
  222. pos = Parse(pos)
  223. pos = Skip(lines, pos, r'\)')
  224. return pos
  225. def Parse(pos):
  226. pos = SkipUntil(lines, pos, r'\(|\)', ')')
  227. if SubString(lines, pos, pos + 1) == '(':
  228. pos = Parse(pos + 1)
  229. pos = Skip(lines, pos, r'\)')
  230. return Parse(pos)
  231. else:
  232. return pos
  233. start = pos.Clone()
  234. pos = ParseInParens(pos)
  235. return MakeToken(lines, start, pos, 'exp')
  236. def RStripNewLineFromToken(token):
  237. if token.value.endswith('\n'):
  238. return Token(token.start, token.end, token.value[:-1], token.token_type)
  239. else:
  240. return token
  241. def TokenizeLines(lines, pos):
  242. while True:
  243. found = FindFirst(lines, TOKEN_TABLE, pos)
  244. if not found:
  245. yield MakeToken(lines, pos, Eof(), 'code')
  246. return
  247. if found.start == pos:
  248. prev_token = None
  249. prev_token_rstripped = None
  250. else:
  251. prev_token = MakeToken(lines, pos, found.start, 'code')
  252. prev_token_rstripped = RStripNewLineFromToken(prev_token)
  253. if found.token_type == '$var':
  254. if prev_token_rstripped:
  255. yield prev_token_rstripped
  256. yield found
  257. id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
  258. yield id_token
  259. pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
  260. eq_token = ParseToken(lines, pos, EQ_REGEX, '=')
  261. yield eq_token
  262. pos = Skip(lines, eq_token.end, r'\s*')
  263. if SubString(lines, pos, pos + 2) != '[[':
  264. exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp')
  265. yield exp_token
  266. pos = Cursor(exp_token.end.line + 1, 0)
  267. elif found.token_type == '$for':
  268. if prev_token_rstripped:
  269. yield prev_token_rstripped
  270. yield found
  271. id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
  272. yield id_token
  273. pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX)
  274. elif found.token_type == '$range':
  275. if prev_token_rstripped:
  276. yield prev_token_rstripped
  277. yield found
  278. id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
  279. yield id_token
  280. pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
  281. dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..')
  282. yield MakeToken(lines, pos, dots_pos, 'exp')
  283. yield MakeToken(lines, dots_pos, dots_pos + 2, '..')
  284. pos = dots_pos + 2
  285. new_pos = Cursor(pos.line + 1, 0)
  286. yield MakeToken(lines, pos, new_pos, 'exp')
  287. pos = new_pos
  288. elif found.token_type == '$':
  289. if prev_token:
  290. yield prev_token
  291. yield found
  292. exp_token = ParseExpTokenInParens(lines, found.end)
  293. yield exp_token
  294. pos = exp_token.end
  295. elif (found.token_type == ']]' or found.token_type == '$if' or
  296. found.token_type == '$elif' or found.token_type == '$else'):
  297. if prev_token_rstripped:
  298. yield prev_token_rstripped
  299. yield found
  300. pos = found.end
  301. else:
  302. if prev_token:
  303. yield prev_token
  304. yield found
  305. pos = found.end
  306. def Tokenize(s):
  307. """A generator that yields the tokens in the given string."""
  308. if s != '':
  309. lines = s.splitlines(True)
  310. for token in TokenizeLines(lines, Cursor(0, 0)):
  311. yield token
  312. class CodeNode:
  313. def __init__(self, atomic_code_list=None):
  314. self.atomic_code = atomic_code_list
  315. class VarNode:
  316. def __init__(self, identifier=None, atomic_code=None):
  317. self.identifier = identifier
  318. self.atomic_code = atomic_code
  319. class RangeNode:
  320. def __init__(self, identifier=None, exp1=None, exp2=None):
  321. self.identifier = identifier
  322. self.exp1 = exp1
  323. self.exp2 = exp2
  324. class ForNode:
  325. def __init__(self, identifier=None, sep=None, code=None):
  326. self.identifier = identifier
  327. self.sep = sep
  328. self.code = code
  329. class ElseNode:
  330. def __init__(self, else_branch=None):
  331. self.else_branch = else_branch
  332. class IfNode:
  333. def __init__(self, exp=None, then_branch=None, else_branch=None):
  334. self.exp = exp
  335. self.then_branch = then_branch
  336. self.else_branch = else_branch
  337. class RawCodeNode:
  338. def __init__(self, token=None):
  339. self.raw_code = token
  340. class LiteralDollarNode:
  341. def __init__(self, token):
  342. self.token = token
  343. class ExpNode:
  344. def __init__(self, token, python_exp):
  345. self.token = token
  346. self.python_exp = python_exp
  347. def PopFront(a_list):
  348. head = a_list[0]
  349. a_list[:1] = []
  350. return head
  351. def PushFront(a_list, elem):
  352. a_list[:0] = [elem]
  353. def PopToken(a_list, token_type=None):
  354. token = PopFront(a_list)
  355. if token_type is not None and token.token_type != token_type:
  356. print('ERROR: %s expected at %s' % (token_type, token.start))
  357. print('ERROR: %s found instead' % (token,))
  358. sys.exit(1)
  359. return token
  360. def PeekToken(a_list):
  361. if not a_list:
  362. return None
  363. return a_list[0]
  364. def ParseExpNode(token):
  365. python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value)
  366. return ExpNode(token, python_exp)
  367. def ParseElseNode(tokens):
  368. def Pop(token_type=None):
  369. return PopToken(tokens, token_type)
  370. next = PeekToken(tokens)
  371. if not next:
  372. return None
  373. if next.token_type == '$else':
  374. Pop('$else')
  375. Pop('[[')
  376. code_node = ParseCodeNode(tokens)
  377. Pop(']]')
  378. return code_node
  379. elif next.token_type == '$elif':
  380. Pop('$elif')
  381. exp = Pop('code')
  382. Pop('[[')
  383. code_node = ParseCodeNode(tokens)
  384. Pop(']]')
  385. inner_else_node = ParseElseNode(tokens)
  386. return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)])
  387. elif not next.value.strip():
  388. Pop('code')
  389. return ParseElseNode(tokens)
  390. else:
  391. return None
  392. def ParseAtomicCodeNode(tokens):
  393. def Pop(token_type=None):
  394. return PopToken(tokens, token_type)
  395. head = PopFront(tokens)
  396. t = head.token_type
  397. if t == 'code':
  398. return RawCodeNode(head)
  399. elif t == '$var':
  400. id_token = Pop('id')
  401. Pop('=')
  402. next = PeekToken(tokens)
  403. if next.token_type == 'exp':
  404. exp_token = Pop()
  405. return VarNode(id_token, ParseExpNode(exp_token))
  406. Pop('[[')
  407. code_node = ParseCodeNode(tokens)
  408. Pop(']]')
  409. return VarNode(id_token, code_node)
  410. elif t == '$for':
  411. id_token = Pop('id')
  412. next_token = PeekToken(tokens)
  413. if next_token.token_type == 'code':
  414. sep_token = next_token
  415. Pop('code')
  416. else:
  417. sep_token = None
  418. Pop('[[')
  419. code_node = ParseCodeNode(tokens)
  420. Pop(']]')
  421. return ForNode(id_token, sep_token, code_node)
  422. elif t == '$if':
  423. exp_token = Pop('code')
  424. Pop('[[')
  425. code_node = ParseCodeNode(tokens)
  426. Pop(']]')
  427. else_node = ParseElseNode(tokens)
  428. return IfNode(ParseExpNode(exp_token), code_node, else_node)
  429. elif t == '$range':
  430. id_token = Pop('id')
  431. exp1_token = Pop('exp')
  432. Pop('..')
  433. exp2_token = Pop('exp')
  434. return RangeNode(id_token, ParseExpNode(exp1_token),
  435. ParseExpNode(exp2_token))
  436. elif t == '$id':
  437. return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id'))
  438. elif t == '$($)':
  439. return LiteralDollarNode(head)
  440. elif t == '$':
  441. exp_token = Pop('exp')
  442. return ParseExpNode(exp_token)
  443. elif t == '[[':
  444. code_node = ParseCodeNode(tokens)
  445. Pop(']]')
  446. return code_node
  447. else:
  448. PushFront(tokens, head)
  449. return None
  450. def ParseCodeNode(tokens):
  451. atomic_code_list = []
  452. while True:
  453. if not tokens:
  454. break
  455. atomic_code_node = ParseAtomicCodeNode(tokens)
  456. if atomic_code_node:
  457. atomic_code_list.append(atomic_code_node)
  458. else:
  459. break
  460. return CodeNode(atomic_code_list)
  461. def ParseToAST(pump_src_text):
  462. """Convert the given Pump source text into an AST."""
  463. tokens = list(Tokenize(pump_src_text))
  464. code_node = ParseCodeNode(tokens)
  465. return code_node
  466. class Env:
  467. def __init__(self):
  468. self.variables = []
  469. self.ranges = []
  470. def Clone(self):
  471. clone = Env()
  472. clone.variables = self.variables[:]
  473. clone.ranges = self.ranges[:]
  474. return clone
  475. def PushVariable(self, var, value):
  476. # If value looks like an int, store it as an int.
  477. try:
  478. int_value = int(value)
  479. if ('%s' % int_value) == value:
  480. value = int_value
  481. except Exception:
  482. pass
  483. self.variables[:0] = [(var, value)]
  484. def PopVariable(self):
  485. self.variables[:1] = []
  486. def PushRange(self, var, lower, upper):
  487. self.ranges[:0] = [(var, lower, upper)]
  488. def PopRange(self):
  489. self.ranges[:1] = []
  490. def GetValue(self, identifier):
  491. for (var, value) in self.variables:
  492. if identifier == var:
  493. return value
  494. print('ERROR: meta variable %s is undefined.' % (identifier,))
  495. sys.exit(1)
  496. def EvalExp(self, exp):
  497. try:
  498. result = eval(exp.python_exp)
  499. except Exception as e: # pylint: disable=broad-except
  500. print('ERROR: caught exception %s: %s' % (e.__class__.__name__, e))
  501. print('ERROR: failed to evaluate meta expression %s at %s' %
  502. (exp.python_exp, exp.token.start))
  503. sys.exit(1)
  504. return result
  505. def GetRange(self, identifier):
  506. for (var, lower, upper) in self.ranges:
  507. if identifier == var:
  508. return (lower, upper)
  509. print('ERROR: range %s is undefined.' % (identifier,))
  510. sys.exit(1)
  511. class Output:
  512. def __init__(self):
  513. self.string = ''
  514. def GetLastLine(self):
  515. index = self.string.rfind('\n')
  516. if index < 0:
  517. return ''
  518. return self.string[index + 1:]
  519. def Append(self, s):
  520. self.string += s
  521. def RunAtomicCode(env, node, output):
  522. if isinstance(node, VarNode):
  523. identifier = node.identifier.value.strip()
  524. result = Output()
  525. RunAtomicCode(env.Clone(), node.atomic_code, result)
  526. value = result.string
  527. env.PushVariable(identifier, value)
  528. elif isinstance(node, RangeNode):
  529. identifier = node.identifier.value.strip()
  530. lower = int(env.EvalExp(node.exp1))
  531. upper = int(env.EvalExp(node.exp2))
  532. env.PushRange(identifier, lower, upper)
  533. elif isinstance(node, ForNode):
  534. identifier = node.identifier.value.strip()
  535. if node.sep is None:
  536. sep = ''
  537. else:
  538. sep = node.sep.value
  539. (lower, upper) = env.GetRange(identifier)
  540. for i in range(lower, upper + 1):
  541. new_env = env.Clone()
  542. new_env.PushVariable(identifier, i)
  543. RunCode(new_env, node.code, output)
  544. if i != upper:
  545. output.Append(sep)
  546. elif isinstance(node, RawCodeNode):
  547. output.Append(node.raw_code.value)
  548. elif isinstance(node, IfNode):
  549. cond = env.EvalExp(node.exp)
  550. if cond:
  551. RunCode(env.Clone(), node.then_branch, output)
  552. elif node.else_branch is not None:
  553. RunCode(env.Clone(), node.else_branch, output)
  554. elif isinstance(node, ExpNode):
  555. value = env.EvalExp(node)
  556. output.Append('%s' % (value,))
  557. elif isinstance(node, LiteralDollarNode):
  558. output.Append('$')
  559. elif isinstance(node, CodeNode):
  560. RunCode(env.Clone(), node, output)
  561. else:
  562. print('BAD')
  563. print(node)
  564. sys.exit(1)
  565. def RunCode(env, code_node, output):
  566. for atomic_code in code_node.atomic_code:
  567. RunAtomicCode(env, atomic_code, output)
  568. def IsSingleLineComment(cur_line):
  569. return '//' in cur_line
  570. def IsInPreprocessorDirective(prev_lines, cur_line):
  571. if cur_line.lstrip().startswith('#'):
  572. return True
  573. return prev_lines and prev_lines[-1].endswith('\\')
  574. def WrapComment(line, output):
  575. loc = line.find('//')
  576. before_comment = line[:loc].rstrip()
  577. if before_comment == '':
  578. indent = loc
  579. else:
  580. output.append(before_comment)
  581. indent = len(before_comment) - len(before_comment.lstrip())
  582. prefix = indent*' ' + '// '
  583. max_len = 80 - len(prefix)
  584. comment = line[loc + 2:].strip()
  585. segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != '']
  586. cur_line = ''
  587. for seg in segs:
  588. if len((cur_line + seg).rstrip()) < max_len:
  589. cur_line += seg
  590. else:
  591. if cur_line.strip() != '':
  592. output.append(prefix + cur_line.rstrip())
  593. cur_line = seg.lstrip()
  594. if cur_line.strip() != '':
  595. output.append(prefix + cur_line.strip())
  596. def WrapCode(line, line_concat, output):
  597. indent = len(line) - len(line.lstrip())
  598. prefix = indent*' ' # Prefix of the current line
  599. max_len = 80 - indent - len(line_concat) # Maximum length of the current line
  600. new_prefix = prefix + 4*' ' # Prefix of a continuation line
  601. new_max_len = max_len - 4 # Maximum length of a continuation line
  602. # Prefers to wrap a line after a ',' or ';'.
  603. segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != '']
  604. cur_line = '' # The current line without leading spaces.
  605. for seg in segs:
  606. # If the line is still too long, wrap at a space.
  607. while cur_line == '' and len(seg.strip()) > max_len:
  608. seg = seg.lstrip()
  609. split_at = seg.rfind(' ', 0, max_len)
  610. output.append(prefix + seg[:split_at].strip() + line_concat)
  611. seg = seg[split_at + 1:]
  612. prefix = new_prefix
  613. max_len = new_max_len
  614. if len((cur_line + seg).rstrip()) < max_len:
  615. cur_line = (cur_line + seg).lstrip()
  616. else:
  617. output.append(prefix + cur_line.rstrip() + line_concat)
  618. prefix = new_prefix
  619. max_len = new_max_len
  620. cur_line = seg.lstrip()
  621. if cur_line.strip() != '':
  622. output.append(prefix + cur_line.strip())
  623. def WrapPreprocessorDirective(line, output):
  624. WrapCode(line, ' \\', output)
  625. def WrapPlainCode(line, output):
  626. WrapCode(line, '', output)
  627. def IsMultiLineIWYUPragma(line):
  628. return re.search(r'/\* IWYU pragma: ', line)
  629. def IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
  630. return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or
  631. re.match(r'^#include\s', line) or
  632. # Don't break IWYU pragmas, either; that causes iwyu.py problems.
  633. re.search(r'// IWYU pragma: ', line))
  634. def WrapLongLine(line, output):
  635. line = line.rstrip()
  636. if len(line) <= 80:
  637. output.append(line)
  638. elif IsSingleLineComment(line):
  639. if IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
  640. # The style guide made an exception to allow long header guard lines,
  641. # includes and IWYU pragmas.
  642. output.append(line)
  643. else:
  644. WrapComment(line, output)
  645. elif IsInPreprocessorDirective(output, line):
  646. if IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
  647. # The style guide made an exception to allow long header guard lines,
  648. # includes and IWYU pragmas.
  649. output.append(line)
  650. else:
  651. WrapPreprocessorDirective(line, output)
  652. elif IsMultiLineIWYUPragma(line):
  653. output.append(line)
  654. else:
  655. WrapPlainCode(line, output)
  656. def BeautifyCode(string):
  657. lines = string.splitlines()
  658. output = []
  659. for line in lines:
  660. WrapLongLine(line, output)
  661. output2 = [line.rstrip() for line in output]
  662. return '\n'.join(output2) + '\n'
  663. def ConvertFromPumpSource(src_text):
  664. """Return the text generated from the given Pump source text."""
  665. ast = ParseToAST(StripMetaComments(src_text))
  666. output = Output()
  667. RunCode(Env(), ast, output)
  668. return BeautifyCode(output.string)
  669. def main(argv):
  670. if len(argv) == 1:
  671. print(__doc__)
  672. sys.exit(1)
  673. file_path = argv[-1]
  674. output_str = ConvertFromPumpSource(io.open(file_path, 'r').read())
  675. if file_path.endswith('.pump'):
  676. output_file_path = file_path[:-5]
  677. else:
  678. output_file_path = '-'
  679. if output_file_path == '-':
  680. print(output_str,)
  681. else:
  682. output_file = io.open(output_file_path, 'w')
  683. output_file.write(u'// This file was GENERATED by command:\n')
  684. output_file.write(u'// %s %s\n' %
  685. (os.path.basename(__file__), os.path.basename(file_path)))
  686. output_file.write(u'// DO NOT EDIT BY HAND!!!\n\n')
  687. output_file.write(output_str)
  688. output_file.close()
  689. if __name__ == '__main__':
  690. main(sys.argv)