patchstream.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2011 The Chromium OS Authors.
  3. #
  4. """Handles parsing a stream of commits/emails from 'git log' or other source"""
  5. import collections
  6. import datetime
  7. import io
  8. import math
  9. import os
  10. import re
  11. import queue
  12. import shutil
  13. import tempfile
  14. from patman import commit
  15. from patman import gitutil
  16. from patman.series import Series
  17. from u_boot_pylib import command
  18. # Tags that we detect and remove
  19. RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:'
  20. r'|Reviewed-on:|Commit-\w*:')
  21. # Lines which are allowed after a TEST= line
  22. RE_ALLOWED_AFTER_TEST = re.compile('^Signed-off-by:')
  23. # Signoffs
  24. RE_SIGNOFF = re.compile('^Signed-off-by: *(.*)')
  25. # Cover letter tag
  26. RE_COVER = re.compile('^Cover-([a-z-]*): *(.*)')
  27. # Patch series tag
  28. RE_SERIES_TAG = re.compile('^Series-([a-z-]*): *(.*)')
  29. # Change-Id will be used to generate the Message-Id and then be stripped
  30. RE_CHANGE_ID = re.compile('^Change-Id: *(.*)')
  31. # Commit series tag
  32. RE_COMMIT_TAG = re.compile('^Commit-([a-z-]*): *(.*)')
  33. # Commit tags that we want to collect and keep
  34. RE_TAG = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc|Fixes): (.*)')
  35. # The start of a new commit in the git log
  36. RE_COMMIT = re.compile('^commit ([0-9a-f]*)$')
  37. # We detect these since checkpatch doesn't always do it
  38. RE_SPACE_BEFORE_TAB = re.compile('^[+].* \t')
  39. # Match indented lines for changes
  40. RE_LEADING_WHITESPACE = re.compile(r'^\s')
  41. # Detect a 'diff' line
  42. RE_DIFF = re.compile(r'^>.*diff --git a/(.*) b/(.*)$')
  43. # Detect a context line, like '> @@ -153,8 +153,13 @@ CheckPatch
  44. RE_LINE = re.compile(r'>.*@@ \-(\d+),\d+ \+(\d+),\d+ @@ *(.*)')
  45. # Detect line with invalid TAG
  46. RE_INV_TAG = re.compile('^Serie-([a-z-]*): *(.*)')
  47. # States we can be in - can we use range() and still have comments?
  48. STATE_MSG_HEADER = 0 # Still in the message header
  49. STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
  50. STATE_PATCH_HEADER = 2 # In patch header (after the subject)
  51. STATE_DIFFS = 3 # In the diff part (past --- line)
  52. class PatchStream:
  53. """Class for detecting/injecting tags in a patch or series of patches
  54. We support processing the output of 'git log' to read out the tags we
  55. are interested in. We can also process a patch file in order to remove
  56. unwanted tags or inject additional ones. These correspond to the two
  57. phases of processing.
  58. """
  59. def __init__(self, series, is_log=False):
  60. self.skip_blank = False # True to skip a single blank line
  61. self.found_test = False # Found a TEST= line
  62. self.lines_after_test = 0 # Number of lines found after TEST=
  63. self.linenum = 1 # Output line number we are up to
  64. self.in_section = None # Name of start...END section we are in
  65. self.notes = [] # Series notes
  66. self.section = [] # The current section...END section
  67. self.series = series # Info about the patch series
  68. self.is_log = is_log # True if indent like git log
  69. self.in_change = None # Name of the change list we are in
  70. self.change_version = 0 # Non-zero if we are in a change list
  71. self.change_lines = [] # Lines of the current change
  72. self.blank_count = 0 # Number of blank lines stored up
  73. self.state = STATE_MSG_HEADER # What state are we in?
  74. self.commit = None # Current commit
  75. # List of unquoted test blocks, each a list of str lines
  76. self.snippets = []
  77. self.cur_diff = None # Last 'diff' line seen (str)
  78. self.cur_line = None # Last context (@@) line seen (str)
  79. self.recent_diff = None # 'diff' line for current snippet (str)
  80. self.recent_line = None # '@@' line for current snippet (str)
  81. self.recent_quoted = collections.deque([], 5)
  82. self.recent_unquoted = queue.Queue()
  83. self.was_quoted = None
  84. @staticmethod
  85. def process_text(text, is_comment=False):
  86. """Process some text through this class using a default Commit/Series
  87. Args:
  88. text (str): Text to parse
  89. is_comment (bool): True if this is a comment rather than a patch.
  90. If True, PatchStream doesn't expect a patch subject at the
  91. start, but jumps straight into the body
  92. Returns:
  93. PatchStream: object with results
  94. """
  95. pstrm = PatchStream(Series())
  96. pstrm.commit = commit.Commit(None)
  97. infd = io.StringIO(text)
  98. outfd = io.StringIO()
  99. if is_comment:
  100. pstrm.state = STATE_PATCH_HEADER
  101. pstrm.process_stream(infd, outfd)
  102. return pstrm
  103. def _add_warn(self, warn):
  104. """Add a new warning to report to the user about the current commit
  105. The new warning is added to the current commit if not already present.
  106. Args:
  107. warn (str): Warning to report
  108. Raises:
  109. ValueError: Warning is generated with no commit associated
  110. """
  111. if not self.commit:
  112. print('Warning outside commit: %s' % warn)
  113. elif warn not in self.commit.warn:
  114. self.commit.warn.append(warn)
  115. def _add_to_series(self, line, name, value):
  116. """Add a new Series-xxx tag.
  117. When a Series-xxx tag is detected, we come here to record it, if we
  118. are scanning a 'git log'.
  119. Args:
  120. line (str): Source line containing tag (useful for debug/error
  121. messages)
  122. name (str): Tag name (part after 'Series-')
  123. value (str): Tag value (part after 'Series-xxx: ')
  124. """
  125. if name == 'notes':
  126. self.in_section = name
  127. self.skip_blank = False
  128. if self.is_log:
  129. warn = self.series.AddTag(self.commit, line, name, value)
  130. if warn:
  131. self.commit.warn.append(warn)
  132. def _add_to_commit(self, name):
  133. """Add a new Commit-xxx tag.
  134. When a Commit-xxx tag is detected, we come here to record it.
  135. Args:
  136. name (str): Tag name (part after 'Commit-')
  137. """
  138. if name == 'notes':
  139. self.in_section = 'commit-' + name
  140. self.skip_blank = False
  141. def _add_commit_rtag(self, rtag_type, who):
  142. """Add a response tag to the current commit
  143. Args:
  144. rtag_type (str): rtag type (e.g. 'Reviewed-by')
  145. who (str): Person who gave that rtag, e.g.
  146. 'Fred Bloggs <fred@bloggs.org>'
  147. """
  148. self.commit.add_rtag(rtag_type, who)
  149. def _close_commit(self):
  150. """Save the current commit into our commit list, and reset our state"""
  151. if self.commit and self.is_log:
  152. self.series.AddCommit(self.commit)
  153. self.commit = None
  154. # If 'END' is missing in a 'Cover-letter' section, and that section
  155. # happens to show up at the very end of the commit message, this is
  156. # the chance for us to fix it up.
  157. if self.in_section == 'cover' and self.is_log:
  158. self.series.cover = self.section
  159. self.in_section = None
  160. self.skip_blank = True
  161. self.section = []
  162. self.cur_diff = None
  163. self.recent_diff = None
  164. self.recent_line = None
  165. def _parse_version(self, value, line):
  166. """Parse a version from a *-changes tag
  167. Args:
  168. value (str): Tag value (part after 'xxx-changes: '
  169. line (str): Source line containing tag
  170. Returns:
  171. int: The version as an integer
  172. Raises:
  173. ValueError: the value cannot be converted
  174. """
  175. try:
  176. return int(value)
  177. except ValueError:
  178. raise ValueError("%s: Cannot decode version info '%s'" %
  179. (self.commit.hash, line))
  180. def _finalise_change(self):
  181. """_finalise a (multi-line) change and add it to the series or commit"""
  182. if not self.change_lines:
  183. return
  184. change = '\n'.join(self.change_lines)
  185. if self.in_change == 'Series':
  186. self.series.AddChange(self.change_version, self.commit, change)
  187. elif self.in_change == 'Cover':
  188. self.series.AddChange(self.change_version, None, change)
  189. elif self.in_change == 'Commit':
  190. self.commit.add_change(self.change_version, change)
  191. self.change_lines = []
  192. def _finalise_snippet(self):
  193. """Finish off a snippet and add it to the list
  194. This is called when we get to the end of a snippet, i.e. the we enter
  195. the next block of quoted text:
  196. This is a comment from someone.
  197. Something else
  198. > Now we have some code <----- end of snippet
  199. > more code
  200. Now a comment about the above code
  201. This adds the snippet to our list
  202. """
  203. quoted_lines = []
  204. while self.recent_quoted:
  205. quoted_lines.append(self.recent_quoted.popleft())
  206. unquoted_lines = []
  207. valid = False
  208. while not self.recent_unquoted.empty():
  209. text = self.recent_unquoted.get()
  210. if not (text.startswith('On ') and text.endswith('wrote:')):
  211. unquoted_lines.append(text)
  212. if text:
  213. valid = True
  214. if valid:
  215. lines = []
  216. if self.recent_diff:
  217. lines.append('> File: %s' % self.recent_diff)
  218. if self.recent_line:
  219. out = '> Line: %s / %s' % self.recent_line[:2]
  220. if self.recent_line[2]:
  221. out += ': %s' % self.recent_line[2]
  222. lines.append(out)
  223. lines += quoted_lines + unquoted_lines
  224. if lines:
  225. self.snippets.append(lines)
  226. def process_line(self, line):
  227. """Process a single line of a patch file or commit log
  228. This process a line and returns a list of lines to output. The list
  229. may be empty or may contain multiple output lines.
  230. This is where all the complicated logic is located. The class's
  231. state is used to move between different states and detect things
  232. properly.
  233. We can be in one of two modes:
  234. self.is_log == True: This is 'git log' mode, where most output is
  235. indented by 4 characters and we are scanning for tags
  236. self.is_log == False: This is 'patch' mode, where we already have
  237. all the tags, and are processing patches to remove junk we
  238. don't want, and add things we think are required.
  239. Args:
  240. line (str): text line to process
  241. Returns:
  242. list: list of output lines, or [] if nothing should be output
  243. Raises:
  244. ValueError: a fatal error occurred while parsing, e.g. an END
  245. without a starting tag, or two commits with two change IDs
  246. """
  247. # Initially we have no output. Prepare the input line string
  248. out = []
  249. line = line.rstrip('\n')
  250. commit_match = RE_COMMIT.match(line) if self.is_log else None
  251. if self.is_log:
  252. if line[:4] == ' ':
  253. line = line[4:]
  254. # Handle state transition and skipping blank lines
  255. series_tag_match = RE_SERIES_TAG.match(line)
  256. change_id_match = RE_CHANGE_ID.match(line)
  257. commit_tag_match = RE_COMMIT_TAG.match(line)
  258. cover_match = RE_COVER.match(line)
  259. signoff_match = RE_SIGNOFF.match(line)
  260. leading_whitespace_match = RE_LEADING_WHITESPACE.match(line)
  261. diff_match = RE_DIFF.match(line)
  262. line_match = RE_LINE.match(line)
  263. invalid_match = RE_INV_TAG.match(line)
  264. tag_match = None
  265. if self.state == STATE_PATCH_HEADER:
  266. tag_match = RE_TAG.match(line)
  267. is_blank = not line.strip()
  268. if is_blank:
  269. if (self.state == STATE_MSG_HEADER
  270. or self.state == STATE_PATCH_SUBJECT):
  271. self.state += 1
  272. # We don't have a subject in the text stream of patch files
  273. # It has its own line with a Subject: tag
  274. if not self.is_log and self.state == STATE_PATCH_SUBJECT:
  275. self.state += 1
  276. elif commit_match:
  277. self.state = STATE_MSG_HEADER
  278. # If a tag is detected, or a new commit starts
  279. if series_tag_match or commit_tag_match or change_id_match or \
  280. cover_match or signoff_match or self.state == STATE_MSG_HEADER:
  281. # but we are already in a section, this means 'END' is missing
  282. # for that section, fix it up.
  283. if self.in_section:
  284. self._add_warn("Missing 'END' in section '%s'" % self.in_section)
  285. if self.in_section == 'cover':
  286. self.series.cover = self.section
  287. elif self.in_section == 'notes':
  288. if self.is_log:
  289. self.series.notes += self.section
  290. elif self.in_section == 'commit-notes':
  291. if self.is_log:
  292. self.commit.notes += self.section
  293. else:
  294. # This should not happen
  295. raise ValueError("Unknown section '%s'" % self.in_section)
  296. self.in_section = None
  297. self.skip_blank = True
  298. self.section = []
  299. # but we are already in a change list, that means a blank line
  300. # is missing, fix it up.
  301. if self.in_change:
  302. self._add_warn("Missing 'blank line' in section '%s-changes'" %
  303. self.in_change)
  304. self._finalise_change()
  305. self.in_change = None
  306. self.change_version = 0
  307. # If we are in a section, keep collecting lines until we see END
  308. if self.in_section:
  309. if line == 'END':
  310. if self.in_section == 'cover':
  311. self.series.cover = self.section
  312. elif self.in_section == 'notes':
  313. if self.is_log:
  314. self.series.notes += self.section
  315. elif self.in_section == 'commit-notes':
  316. if self.is_log:
  317. self.commit.notes += self.section
  318. else:
  319. # This should not happen
  320. raise ValueError("Unknown section '%s'" % self.in_section)
  321. self.in_section = None
  322. self.skip_blank = True
  323. self.section = []
  324. else:
  325. self.section.append(line)
  326. # If we are not in a section, it is an unexpected END
  327. elif line == 'END':
  328. raise ValueError("'END' wihout section")
  329. # Detect the commit subject
  330. elif not is_blank and self.state == STATE_PATCH_SUBJECT:
  331. self.commit.subject = line
  332. # Detect the tags we want to remove, and skip blank lines
  333. elif RE_REMOVE.match(line) and not commit_tag_match:
  334. self.skip_blank = True
  335. # TEST= should be the last thing in the commit, so remove
  336. # everything after it
  337. if line.startswith('TEST='):
  338. self.found_test = True
  339. elif self.skip_blank and is_blank:
  340. self.skip_blank = False
  341. # Detect Cover-xxx tags
  342. elif cover_match:
  343. name = cover_match.group(1)
  344. value = cover_match.group(2)
  345. if name == 'letter':
  346. self.in_section = 'cover'
  347. self.skip_blank = False
  348. elif name == 'letter-cc':
  349. self._add_to_series(line, 'cover-cc', value)
  350. elif name == 'changes':
  351. self.in_change = 'Cover'
  352. self.change_version = self._parse_version(value, line)
  353. # If we are in a change list, key collected lines until a blank one
  354. elif self.in_change:
  355. if is_blank:
  356. # Blank line ends this change list
  357. self._finalise_change()
  358. self.in_change = None
  359. self.change_version = 0
  360. elif line == '---':
  361. self._finalise_change()
  362. self.in_change = None
  363. self.change_version = 0
  364. out = self.process_line(line)
  365. elif self.is_log:
  366. if not leading_whitespace_match:
  367. self._finalise_change()
  368. self.change_lines.append(line)
  369. self.skip_blank = False
  370. # Detect Series-xxx tags
  371. elif series_tag_match:
  372. name = series_tag_match.group(1)
  373. value = series_tag_match.group(2)
  374. if name == 'changes':
  375. # value is the version number: e.g. 1, or 2
  376. self.in_change = 'Series'
  377. self.change_version = self._parse_version(value, line)
  378. else:
  379. self._add_to_series(line, name, value)
  380. self.skip_blank = True
  381. # Detect Change-Id tags
  382. elif change_id_match:
  383. value = change_id_match.group(1)
  384. if self.is_log:
  385. if self.commit.change_id:
  386. raise ValueError(
  387. "%s: Two Change-Ids: '%s' vs. '%s'" %
  388. (self.commit.hash, self.commit.change_id, value))
  389. self.commit.change_id = value
  390. self.skip_blank = True
  391. # Detect Commit-xxx tags
  392. elif commit_tag_match:
  393. name = commit_tag_match.group(1)
  394. value = commit_tag_match.group(2)
  395. if name == 'notes':
  396. self._add_to_commit(name)
  397. self.skip_blank = True
  398. elif name == 'changes':
  399. self.in_change = 'Commit'
  400. self.change_version = self._parse_version(value, line)
  401. else:
  402. self._add_warn('Line %d: Ignoring Commit-%s' %
  403. (self.linenum, name))
  404. # Detect invalid tags
  405. elif invalid_match:
  406. raise ValueError("Line %d: Invalid tag = '%s'" %
  407. (self.linenum, line))
  408. # Detect the start of a new commit
  409. elif commit_match:
  410. self._close_commit()
  411. self.commit = commit.Commit(commit_match.group(1))
  412. # Detect tags in the commit message
  413. elif tag_match:
  414. rtag_type, who = tag_match.groups()
  415. self._add_commit_rtag(rtag_type, who)
  416. # Remove Tested-by self, since few will take much notice
  417. if (rtag_type == 'Tested-by' and
  418. who.find(os.getenv('USER') + '@') != -1):
  419. self._add_warn("Ignoring '%s'" % line)
  420. elif rtag_type == 'Patch-cc':
  421. self.commit.add_cc(who.split(','))
  422. else:
  423. out = [line]
  424. # Suppress duplicate signoffs
  425. elif signoff_match:
  426. if (self.is_log or not self.commit or
  427. self.commit.check_duplicate_signoff(signoff_match.group(1))):
  428. out = [line]
  429. # Well that means this is an ordinary line
  430. else:
  431. # Look for space before tab
  432. mat = RE_SPACE_BEFORE_TAB.match(line)
  433. if mat:
  434. self._add_warn('Line %d/%d has space before tab' %
  435. (self.linenum, mat.start()))
  436. # OK, we have a valid non-blank line
  437. out = [line]
  438. self.linenum += 1
  439. self.skip_blank = False
  440. if diff_match:
  441. self.cur_diff = diff_match.group(1)
  442. # If this is quoted, keep recent lines
  443. if not diff_match and self.linenum > 1 and line:
  444. if line.startswith('>'):
  445. if not self.was_quoted:
  446. self._finalise_snippet()
  447. self.recent_line = None
  448. if not line_match:
  449. self.recent_quoted.append(line)
  450. self.was_quoted = True
  451. self.recent_diff = self.cur_diff
  452. else:
  453. self.recent_unquoted.put(line)
  454. self.was_quoted = False
  455. if line_match:
  456. self.recent_line = line_match.groups()
  457. if self.state == STATE_DIFFS:
  458. pass
  459. # If this is the start of the diffs section, emit our tags and
  460. # change log
  461. elif line == '---':
  462. self.state = STATE_DIFFS
  463. # Output the tags (signoff first), then change list
  464. out = []
  465. log = self.series.MakeChangeLog(self.commit)
  466. out += [line]
  467. if self.commit:
  468. out += self.commit.notes
  469. out += [''] + log
  470. elif self.found_test:
  471. if not RE_ALLOWED_AFTER_TEST.match(line):
  472. self.lines_after_test += 1
  473. return out
  474. def finalise(self):
  475. """Close out processing of this patch stream"""
  476. self._finalise_snippet()
  477. self._finalise_change()
  478. self._close_commit()
  479. if self.lines_after_test:
  480. self._add_warn('Found %d lines after TEST=' % self.lines_after_test)
  481. def _write_message_id(self, outfd):
  482. """Write the Message-Id into the output.
  483. This is based on the Change-Id in the original patch, the version,
  484. and the prefix.
  485. Args:
  486. outfd (io.IOBase): Output stream file object
  487. """
  488. if not self.commit.change_id:
  489. return
  490. # If the count is -1 we're testing, so use a fixed time
  491. if self.commit.count == -1:
  492. time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
  493. else:
  494. time_now = datetime.datetime.now()
  495. # In theory there is email.utils.make_msgid() which would be nice
  496. # to use, but it already produces something way too long and thus
  497. # will produce ugly commit lines if someone throws this into
  498. # a "Link:" tag in the final commit. So (sigh) roll our own.
  499. # Start with the time; presumably we wouldn't send the same series
  500. # with the same Change-Id at the exact same second.
  501. parts = [time_now.strftime("%Y%m%d%H%M%S")]
  502. # These seem like they would be nice to include.
  503. if 'prefix' in self.series:
  504. parts.append(self.series['prefix'])
  505. if 'postfix' in self.series:
  506. parts.append(self.series['postfix'])
  507. if 'version' in self.series:
  508. parts.append("v%s" % self.series['version'])
  509. parts.append(str(self.commit.count + 1))
  510. # The Change-Id must be last, right before the @
  511. parts.append(self.commit.change_id)
  512. # Join parts together with "." and write it out.
  513. outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
  514. def process_stream(self, infd, outfd):
  515. """Copy a stream from infd to outfd, filtering out unwanting things.
  516. This is used to process patch files one at a time.
  517. Args:
  518. infd (io.IOBase): Input stream file object
  519. outfd (io.IOBase): Output stream file object
  520. """
  521. # Extract the filename from each diff, for nice warnings
  522. fname = None
  523. last_fname = None
  524. re_fname = re.compile('diff --git a/(.*) b/.*')
  525. self._write_message_id(outfd)
  526. while True:
  527. line = infd.readline()
  528. if not line:
  529. break
  530. out = self.process_line(line)
  531. # Try to detect blank lines at EOF
  532. for line in out:
  533. match = re_fname.match(line)
  534. if match:
  535. last_fname = fname
  536. fname = match.group(1)
  537. if line == '+':
  538. self.blank_count += 1
  539. else:
  540. if self.blank_count and (line == '-- ' or match):
  541. self._add_warn("Found possible blank line(s) at end of file '%s'" %
  542. last_fname)
  543. outfd.write('+\n' * self.blank_count)
  544. outfd.write(line + '\n')
  545. self.blank_count = 0
  546. self.finalise()
  547. def insert_tags(msg, tags_to_emit):
  548. """Add extra tags to a commit message
  549. The tags are added after an existing block of tags if found, otherwise at
  550. the end.
  551. Args:
  552. msg (str): Commit message
  553. tags_to_emit (list): List of tags to emit, each a str
  554. Returns:
  555. (str) new message
  556. """
  557. out = []
  558. done = False
  559. emit_tags = False
  560. emit_blank = False
  561. for line in msg.splitlines():
  562. if not done:
  563. signoff_match = RE_SIGNOFF.match(line)
  564. tag_match = RE_TAG.match(line)
  565. if tag_match or signoff_match:
  566. emit_tags = True
  567. if emit_tags and not tag_match and not signoff_match:
  568. out += tags_to_emit
  569. emit_tags = False
  570. done = True
  571. emit_blank = not (signoff_match or tag_match)
  572. else:
  573. emit_blank = line
  574. out.append(line)
  575. if not done:
  576. if emit_blank:
  577. out.append('')
  578. out += tags_to_emit
  579. return '\n'.join(out)
  580. def get_list(commit_range, git_dir=None, count=None):
  581. """Get a log of a list of comments
  582. This returns the output of 'git log' for the selected commits
  583. Args:
  584. commit_range (str): Range of commits to count (e.g. 'HEAD..base')
  585. git_dir (str): Path to git repositiory (None to use default)
  586. count (int): Number of commits to list, or None for no limit
  587. Returns
  588. str: String containing the contents of the git log
  589. """
  590. params = gitutil.log_cmd(commit_range, reverse=True, count=count,
  591. git_dir=git_dir)
  592. return command.run_pipe([params], capture=True).stdout
  593. def get_metadata_for_list(commit_range, git_dir=None, count=None,
  594. series=None, allow_overwrite=False):
  595. """Reads out patch series metadata from the commits
  596. This does a 'git log' on the relevant commits and pulls out the tags we
  597. are interested in.
  598. Args:
  599. commit_range (str): Range of commits to count (e.g. 'HEAD..base')
  600. git_dir (str): Path to git repositiory (None to use default)
  601. count (int): Number of commits to list, or None for no limit
  602. series (Series): Object to add information into. By default a new series
  603. is started.
  604. allow_overwrite (bool): Allow tags to overwrite an existing tag
  605. Returns:
  606. Series: Object containing information about the commits.
  607. """
  608. if not series:
  609. series = Series()
  610. series.allow_overwrite = allow_overwrite
  611. stdout = get_list(commit_range, git_dir, count)
  612. pst = PatchStream(series, is_log=True)
  613. for line in stdout.splitlines():
  614. pst.process_line(line)
  615. pst.finalise()
  616. return series
  617. def get_metadata(branch, start, count):
  618. """Reads out patch series metadata from the commits
  619. This does a 'git log' on the relevant commits and pulls out the tags we
  620. are interested in.
  621. Args:
  622. branch (str): Branch to use (None for current branch)
  623. start (int): Commit to start from: 0=branch HEAD, 1=next one, etc.
  624. count (int): Number of commits to list
  625. Returns:
  626. Series: Object containing information about the commits.
  627. """
  628. return get_metadata_for_list(
  629. '%s~%d' % (branch if branch else 'HEAD', start), None, count)
  630. def get_metadata_for_test(text):
  631. """Process metadata from a file containing a git log. Used for tests
  632. Args:
  633. text:
  634. Returns:
  635. Series: Object containing information about the commits.
  636. """
  637. series = Series()
  638. pst = PatchStream(series, is_log=True)
  639. for line in text.splitlines():
  640. pst.process_line(line)
  641. pst.finalise()
  642. return series
  643. def fix_patch(backup_dir, fname, series, cmt):
  644. """Fix up a patch file, by adding/removing as required.
  645. We remove our tags from the patch file, insert changes lists, etc.
  646. The patch file is processed in place, and overwritten.
  647. A backup file is put into backup_dir (if not None).
  648. Args:
  649. backup_dir (str): Path to directory to use to backup the file
  650. fname (str): Filename to patch file to process
  651. series (Series): Series information about this patch set
  652. cmt (Commit): Commit object for this patch file
  653. Return:
  654. list: A list of errors, each str, or [] if all ok.
  655. """
  656. handle, tmpname = tempfile.mkstemp()
  657. outfd = os.fdopen(handle, 'w', encoding='utf-8')
  658. infd = open(fname, 'r', encoding='utf-8')
  659. pst = PatchStream(series)
  660. pst.commit = cmt
  661. pst.process_stream(infd, outfd)
  662. infd.close()
  663. outfd.close()
  664. # Create a backup file if required
  665. if backup_dir:
  666. shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
  667. shutil.move(tmpname, fname)
  668. return cmt.warn
  669. def fix_patches(series, fnames):
  670. """Fix up a list of patches identified by filenames
  671. The patch files are processed in place, and overwritten.
  672. Args:
  673. series (Series): The Series object
  674. fnames (:type: list of str): List of patch files to process
  675. """
  676. # Current workflow creates patches, so we shouldn't need a backup
  677. backup_dir = None #tempfile.mkdtemp('clean-patch')
  678. count = 0
  679. for fname in fnames:
  680. cmt = series.commits[count]
  681. cmt.patch = fname
  682. cmt.count = count
  683. result = fix_patch(backup_dir, fname, series, cmt)
  684. if result:
  685. print('%d warning%s for %s:' %
  686. (len(result), 's' if len(result) > 1 else '', fname))
  687. for warn in result:
  688. print('\t%s' % warn)
  689. print()
  690. count += 1
  691. print('Cleaned %d patch%s' % (count, 'es' if count > 1 else ''))
  692. def insert_cover_letter(fname, series, count):
  693. """Inserts a cover letter with the required info into patch 0
  694. Args:
  695. fname (str): Input / output filename of the cover letter file
  696. series (Series): Series object
  697. count (int): Number of patches in the series
  698. """
  699. fil = open(fname, 'r')
  700. lines = fil.readlines()
  701. fil.close()
  702. fil = open(fname, 'w')
  703. text = series.cover
  704. prefix = series.GetPatchPrefix()
  705. for line in lines:
  706. if line.startswith('Subject:'):
  707. # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
  708. zero_repeat = int(math.log10(count)) + 1
  709. zero = '0' * zero_repeat
  710. line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
  711. # Insert our cover letter
  712. elif line.startswith('*** BLURB HERE ***'):
  713. # First the blurb test
  714. line = '\n'.join(text[1:]) + '\n'
  715. if series.get('notes'):
  716. line += '\n'.join(series.notes) + '\n'
  717. # Now the change list
  718. out = series.MakeChangeLog(None)
  719. line += '\n' + '\n'.join(out)
  720. fil.write(line)
  721. fil.close()