Repository navigation
Expand file tree
/
Copy pathrelease.py
More file actions
1559 lines (1310 loc) · 65.7 KB
/
Copy pathrelease.py
File metadata and controls
1559 lines (1310 loc) · 65.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""socket-patch release-train tooling (docs/release-train/DESIGN.md).
Stdlib-only, one file, so every workflow and routine can run it with a bare
`python3`. Subcommands (PR 1 of the train):
semver validate|compare|core version grammar: X.Y.Z or X.Y.Z-rc.N only
stamp <V> [--check] offline, byte-deterministic version stamp
(Cargo.toml, Cargo.lock, 15 npm manifests,
the npm wrapper lockfile)
npm-lock-check offline: the npm wrapper lockfile agrees with
its package.json beyond the stamped versions
next-version the version the next rc cut would get,
derived from git tags + burned release/*
branches + the [Unreleased] headings
changelog cut|promote|sync-main|check
CHANGELOG.md transforms (see "CHANGELOG
model" below)
sync-main [--check] the rolling release-sync PR's deterministic
part: CHANGELOG sync + stamp of the newest tag
notes --version V GitHub release notes for one section
blockers --base <sha> the release-blocker gate (DESIGN.md §3.5)
Exit status: 0 ok, 1 refused/blocked/drift, 2 usage error.
Version rules
-------------
* Grammar: `X.Y.Z` (stable) or `X.Y.Z-rc.N` (N >= 1, no leading zeros). No
other prerelease or build-metadata forms exist in this project.
Precedence is semver's: X.Y.Z-rc.N < X.Y.Z-rc.(N+1) < X.Y.Z.
* L = the newest stable tag. The bump level comes from the `### ` headings of
[Unreleased] that still hold entries *after* `changelog sync-main` has been
applied in memory (so an unmerged release-sync PR changes nothing):
a heading containing "breaking" or starting "Removed" -> major;
starting "Added" / "Changed" / "Deprecated" -> minor; anything else -> patch.
* core = max(bump(L, level), every pending rc core above L). N = 1 + the
highest rc number of that core over tags AND release/v<core>-rc.* branches
(a number is burned when its branch is created, even if it never ships).
* Major rule: a core whose major is above L's major may only be cut when
that major is listed in APPROVED_MAJORS below, and only L.major + 1 (a major
is never skipped). Adding a major there is a reviewed PR on main, i.e. a
human decision. Once a major's train is in flight (an rc tag M.0.0-rc.N
exists) further breaking entries are absorbed into M.0.0. A breaking
heading that would open an unapproved major is refused with an error
rather than silently shipped as a minor.
* Main's own Cargo/npm version is never read for any of this.
CHANGELOG model
---------------
Sections are `## [Unreleased]` and `## [V] — YYYY-MM-DD`; subsections are
`### Name`. A *block* is a top-level bullet with its continuation lines, a
paragraph, or a fenced code block. Block identity is (subsection name, text
with trailing whitespace stripped); all matching is exact and counts
occurrences (multisets): removing a shipped block removes one occurrence, so
an identical entry written again later is kept. CRLF files stay CRLF.
* cut V: apply sync-main in memory, then move every [Unreleased] block into a
new `## [V] — date` section directly below an empty [Unreleased]. The new
section's `###` subsections are put in canonical order (SUBSECTION_ORDER),
so the cut does not depend on the order main's [Unreleased] happens to have.
* promote rc.K: fold every `[X-rc.N]` section with X-rc.N <= rc.K that is
newer than the newest stable section into one `## [core] — date` section
(oldest rc first; within a subsection later rcs append; rc headers
removed). The fold keeps every occurrence: an entry written in two rcs
(`- Updated dependencies.` twice) appears twice, so [core] is exactly the
multiset sum of the folded rcs, the same count sync-main charges them
against. Later rc sections of the same core (rc.K+1..) are abandoned:
all of their blocks move back into [Unreleased] (before the blocks already
there) and their headers are dropped.
* sync-main (main's CHANGELOG := what it would be had every release-sync PR
merged). For train-era tags (version > TRAIN_FLOOR):
1. each stable tag S whose section main lacks: insert the tag's own [S]
section; its blocks form S's budget (a multiset);
2. each rc section on main whose core already shipped (owner S = the
smallest stable tag >= its core) is dropped. Its blocks (text from
its tag) are charged against S's budget, oldest rc first; the ones
the budget does not cover did not ship and move back into
[Unreleased], oldest rc first and before the blocks already there. A
folded rc returns nothing; an abandoned later rc, or a train rc cut
beside a hotfix of the same core, returns exactly what did not ship.
No ancestry or rc-number rule is involved;
3. what is left of each new S's budget (the shipped blocks main never
got as an rc section, i.e. still in [Unreleased] on an unsynced
main) is removed from [Unreleased], one occurrence each, oldest
first. Charging the rc sections before touching [Unreleased] keeps a
newer identical entry (which did not ship) in place either way;
4. each pending rc tag (no owner yet) whose section main lacks: insert
the tag's section and remove those blocks from [Unreleased].
Only blocks of sections inserted *in this run* are removed from
[Unreleased], so entries added to [Unreleased] later are never touched and
a second run is a no-op. Section text always comes from the tag (the bytes
that shipped); rc-section edits made on main are dropped by the fold.
With entries appended at the end of their subsection (the convention), the
next cut is byte-identical whether or not release-sync PRs merged.
"""
from __future__ import annotations
import argparse
import collections
import copy
import datetime
import functools
import json
import os
import re
import subprocess
import sys
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
# Majors a cut may open. Adding one is the human-reviewed approval of a new
# major line (DESIGN.md D4: 5.0.0 is pre-approved as the train's first release).
APPROVED_MAJORS = (5,)
# Tags at or below this predate the train; their sections are already on main
# and are never synced (v3.3.0, for one, has no CHANGELOG section at all).
TRAIN_FLOOR = "4.0.0"
DEFAULT_REPO = "SocketDev/socket-patch"
BLOCKER_LABEL = "release-blocker"
P1_LABEL = "priority:p1"
API_ROOT = "https://api.github.com"
class ReleaseError(Exception):
"""A refusal: printed as `release.py: error: ...`, exit status 1."""
def _read(path):
"""Exact text (no newline translation), so byte comparisons are exact."""
with open(path, encoding="utf-8", newline="") as f:
return f.read()
def _write(path, text):
with open(path, "w", encoding="utf-8", newline="") as f:
f.write(text)
def _is_crlf(text):
"""A CRLF file (a Windows checkout with core.autocrlf: the repo has no
`* text=auto`, so every text file may arrive CRLF)."""
return text.count("\r\n") * 2 > text.count("\n")
def _with_eol(text, transform):
"""transform(LF text) applied to `text`, keeping its line endings: a
CRLF file is normalized to LF, transformed, and written back CRLF."""
if not _is_crlf(text):
return transform(text)
return transform(text.replace("\r\n", "\n")).replace("\n", "\r\n")
# ── semver ──────────────────────────────────────────────────────────────────
_VERSION_RE = re.compile(
r"^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-rc\.([1-9][0-9]*))?$")
@functools.total_ordering
class Version:
__slots__ = ("major", "minor", "patch", "rc")
def __init__(self, major, minor, patch, rc=None):
self.major, self.minor, self.patch, self.rc = major, minor, patch, rc
@classmethod
def parse(cls, text):
m = _VERSION_RE.match(text or "")
if not m:
raise ReleaseError(
f"'{text}' is not a release version (X.Y.Z or X.Y.Z-rc.N, N >= 1, "
"no leading zeros, no other prerelease or build forms)")
rc = int(m.group(4)) if m.group(4) else None
return cls(int(m.group(1)), int(m.group(2)), int(m.group(3)), rc)
@classmethod
def try_parse(cls, text):
try:
return cls.parse(text)
except ReleaseError:
return None
@property
def is_rc(self):
return self.rc is not None
@property
def core(self):
return Version(self.major, self.minor, self.patch)
def with_rc(self, n):
return Version(self.major, self.minor, self.patch, n)
def bump(self, level):
if level == "major":
return Version(self.major + 1, 0, 0)
if level == "minor":
return Version(self.major, self.minor + 1, 0)
if level == "patch":
return Version(self.major, self.minor, self.patch + 1)
raise ValueError(level)
def _key(self):
# A prerelease sorts below its own core; rc numbers compare numerically.
return (self.major, self.minor, self.patch,
0 if self.is_rc else 1, self.rc or 0)
def __eq__(self, other):
return isinstance(other, Version) and self._key() == other._key()
def __lt__(self, other):
return self._key() < other._key()
def __hash__(self):
return hash(self._key())
def __str__(self):
s = f"{self.major}.{self.minor}.{self.patch}"
return f"{s}-rc.{self.rc}" if self.is_rc else s
__repr__ = __str__
def parse_tag(name):
"""`v5.0.0-rc.1` -> Version; anything else (non-release tags) -> None."""
if not name.startswith("v"):
return None
return Version.try_parse(name[1:])
def parse_release_branch(name):
"""`release/v5.0.0-rc.2` (any remote prefix) -> Version, else None."""
m = re.search(r"(?:^|/)release/v([^/]+)$", name)
return Version.try_parse(m.group(1)) if m else None
# ── git ─────────────────────────────────────────────────────────────────────
# Variables that make git operate on a repository other than the one at
# `cwd` (a hook's GIT_DIR, for one). Git(root) always means root.
_GIT_REPO_ENV = ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY",
"GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_COMMON_DIR", "GIT_NAMESPACE",
"GIT_CEILING_DIRECTORIES", "GIT_PREFIX")
def git_env(extra=None):
env = {k: v for k, v in os.environ.items() if k not in _GIT_REPO_ENV}
env.update(extra or {})
return env
class Git:
def __init__(self, root):
self.root = Path(root)
def run(self, *args, ok_codes=(0,)):
proc = subprocess.run(["git", *args], cwd=self.root, capture_output=True,
text=True, encoding="utf-8", env=git_env())
if proc.returncode not in ok_codes:
raise ReleaseError(f"git {' '.join(args)} failed: {proc.stderr.strip()}")
return proc
def tags(self):
"""{Version: tag name} for every v<release-version> tag."""
out = self.run("tag", "--list", "v*").stdout.split()
return {v: t for t in out for v in [parse_tag(t)] if v is not None}
def release_branches(self, remote=None):
names = self.run("for-each-ref", "--format=%(refname)",
"refs/heads/release/", "refs/remotes/").stdout.split()
if remote:
ls = self.run("ls-remote", "--heads", remote, "refs/heads/release/*").stdout
names += [line.split("\t", 1)[1] for line in ls.splitlines() if "\t" in line]
return sorted({v for n in names for v in [parse_release_branch(n)] if v})
def show(self, rev, path):
return self.run("show", f"{rev}:{path}").stdout
def commit_date(self, rev):
return self.run("log", "-1", "--format=%cI", f"{rev}^{{commit}}").stdout.strip()
class TagSource:
"""What sync-main needs to know about tags: which exist and the
CHANGELOG at each. Backed by git here; tests may hand in the same two
facts without git."""
def __init__(self, git):
self.git = git
self._tags = git.tags()
self._changelogs = {}
@property
def versions(self):
return sorted(self._tags)
def changelog(self, v):
if v not in self._changelogs:
self._changelogs[v] = self.git.show(f"refs/tags/{self._tags[v]}", "CHANGELOG.md")
return self._changelogs[v]
# ── stamp ───────────────────────────────────────────────────────────────────
_TOML_HEADER = re.compile(r"^\[\[?([^\]]+)\]\]?\s*(#.*)?$")
def _stamp_cargo_toml(text, version):
out, table = [], None
hits = {"version": 0, "pin": 0}
for line in text.splitlines(keepends=True):
body = line.rstrip("\r\n")
m = _TOML_HEADER.match(body)
if m:
table = m.group(1).strip()
elif table == "workspace.package" and re.match(r'^version\s*=\s*"[^"]*"\s*$', body):
line = re.sub(r'"[^"]*"', f'"{version}"', line, count=1)
hits["version"] += 1
elif table == "workspace.dependencies" and re.match(r"^socket-patch-core\s*=", body):
line, n = re.subn(r'(\bversion\s*=\s*")=?[^"]*(")', rf"\g<1>={version}\g<2>", line)
hits["pin"] += n
out.append(line)
if hits != {"version": 1, "pin": 1}:
raise ReleaseError(
"Cargo.toml: expected exactly one [workspace.package] version and one "
f"socket-patch-core version pin, found {hits}")
return "".join(out)
def _workspace_members(root, cargo_toml):
"""Package names of workspace members whose version is inherited from
[workspace.package] (only those move with the release version)."""
m = re.search(r"^members\s*=\s*\[(.*?)\]", cargo_toml, re.S | re.M)
if not m:
raise ReleaseError("Cargo.toml: no [workspace] members list")
names = []
for rel in re.findall(r'"([^"]+)"', m.group(1)):
manifest = _read(root / rel / "Cargo.toml")
name = re.search(r'^name\s*=\s*"([^"]+)"', manifest, re.M)
if name and re.search(r"^version\.workspace\s*=\s*true\s*$", manifest, re.M):
names.append(name.group(1))
return names
def _stamp_cargo_lock(text, version, members):
chunks = re.split(r"(?m)^(?=\[\[package\]\]$)", text)
seen = []
for i, chunk in enumerate(chunks):
if not chunk.startswith("[[package]]"):
continue
name = re.search(r'^name = "([^"]+)"$', chunk, re.M)
if not name or name.group(1) not in members or re.search(r"^source = ", chunk, re.M):
continue
chunks[i], n = re.subn(r'(?m)^version = "[^"]*"$', f'version = "{version}"', chunk, count=1)
if n != 1:
raise ReleaseError(f"Cargo.lock: package {name.group(1)} has no version line")
seen.append(name.group(1))
if sorted(seen) != sorted(members):
raise ReleaseError(
f"Cargo.lock: expected one source-less entry per workspace member {sorted(members)}, "
f"found {sorted(seen)} (run `cargo metadata` once to refresh the lock)")
return "".join(chunks)
def _dump_json(obj):
return json.dumps(obj, indent=2, ensure_ascii=False) + "\n"
def _npm_manifests(root):
main = root / "npm" / "socket-patch" / "package.json"
platforms = sorted(root.glob("npm/socket-patch-*/package.json"))
if not main.is_file() or not platforms:
raise ReleaseError("npm/: expected npm/socket-patch and npm/socket-patch-*/ packages")
return [main] + platforms
_PLATFORM_LOCK_KEY = re.compile(r"^node_modules/@socketsecurity/socket-patch-[^/]+$")
def stamp_files(root, version):
"""{relative path: new bytes} for every stamped file whose bytes change.
Pure: reads the tree, writes nothing."""
version = str(Version.parse(str(version)))
root = Path(root)
changes = {}
def put(path, transform):
# Every file keeps its own line endings (a CRLF checkout stays CRLF,
# so `stamp --check` on it is a byte no-op).
old = _read(path)
new = _with_eol(old, transform)
if new != old:
changes[path.relative_to(root).as_posix()] = new
cargo_toml = root / "Cargo.toml"
toml_text = _read(cargo_toml).replace("\r\n", "\n")
put(cargo_toml, lambda text: _stamp_cargo_toml(text, version))
members = _workspace_members(root, toml_text)
put(root / "Cargo.lock", lambda text: _stamp_cargo_lock(text, version, members))
def stamp_manifest(main):
def transform(text):
pkg = json.loads(text)
pkg["version"] = version
if main:
for dep in pkg.get("optionalDependencies", {}):
pkg["optionalDependencies"][dep] = version
return _dump_json(pkg)
return transform
for i, manifest in enumerate(_npm_manifests(root)):
put(manifest, stamp_manifest(i == 0))
def stamp_npm_lock(text):
obj = json.loads(text)
obj["version"] = version
top = obj.get("packages", {}).get("")
if top is None:
raise ReleaseError('package-lock.json: no packages[""] entry')
top["version"] = version
for dep in top.get("optionalDependencies", {}):
top["optionalDependencies"][dep] = version
# Platform entries pin a registry tarball + integrity for one version;
# an entry for any other version is stale and would make `npm ci`
# refuse the lock. Dropping it (instead of re-resolving over the
# network) keeps the stamp offline and deterministic; `npm install`
# resolves it on demand.
obj["packages"] = {k: v for k, v in obj["packages"].items()
if not (_PLATFORM_LOCK_KEY.match(k) and v.get("version") != version)}
return _dump_json(obj)
put(root / "npm" / "socket-patch" / "package-lock.json", stamp_npm_lock)
return changes
_LOCK_TOP_FIELDS = ("name", "version", "dependencies", "devDependencies", "optionalDependencies",
"peerDependencies", "engines", "bin")
_EXACT_VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?$")
def npm_lock_drift(root):
"""Offline coherence of npm/socket-patch/package-lock.json with its
package.json, beyond the version fields `stamp` owns: packages[""] must
repeat the manifest's dependency maps, engines, bin and name, and every
non-optional dependency needs a `node_modules/<name>` entry (at exactly
that version when the manifest pins one). Returns a list of problems.
(This is what the networked `npm install --package-lock-only` drift check
caught before the stamp went offline.)"""
root = Path(root)
pkg = json.loads(_read(root / "npm" / "socket-patch" / "package.json"))
lock = json.loads(_read(root / "npm" / "socket-patch" / "package-lock.json"))
top = (lock.get("packages") or {}).get("")
if top is None:
return ['package-lock.json: no packages[""] entry']
problems = []
for field in _LOCK_TOP_FIELDS:
want, have = pkg.get(field), top.get(field)
if field == "bin" and isinstance(want, str):
want = {pkg.get("name", "").split("/")[-1]: want}
if (want or None) != (have or None):
problems.append(f'package-lock.json packages[""].{field} != package.json {field}')
if lock.get("name") != pkg.get("name"):
problems.append("package-lock.json name != package.json name")
for field in ("dependencies", "devDependencies"):
for dep, spec in sorted((pkg.get(field) or {}).items()):
entry = lock["packages"].get(f"node_modules/{dep}")
if entry is None:
problems.append(f"package-lock.json has no node_modules/{dep} ({field})")
elif _EXACT_VERSION.match(spec) and entry.get("version") != spec:
problems.append(f"package-lock.json node_modules/{dep} is {entry.get('version')}, "
f"package.json {field} pins {spec}")
return problems
def stamp(root, version, check=False):
changes = stamp_files(root, version)
if not check:
for rel, text in changes.items():
_write(Path(root) / rel, text)
return sorted(changes)
# ── CHANGELOG ───────────────────────────────────────────────────────────────
_BULLET = re.compile(r"^(?:[-*+]|[0-9]+[.)])(?:\s|$)")
_FENCE = re.compile(r"^\s*(```|~~~)")
UNRELEASED = "Unreleased"
def _blank(line):
return not line.strip()
def _indented(line):
return line[:1] in (" ", "\t")
class Block:
__slots__ = ("lines", "gap")
def __init__(self, lines, gap=0):
self.lines, self.gap = list(lines), gap
@property
def is_bullet(self):
return bool(_BULLET.match(self.lines[0]))
@property
def text(self):
return "\n".join(line.rstrip() for line in self.lines)
class Sub:
"""A subsection (`### Name`), or a section's preamble when heading is None."""
__slots__ = ("heading", "lead", "blocks", "trail")
def __init__(self, heading, lead=0, blocks=(), trail=0):
self.heading, self.lead, self.blocks, self.trail = heading, lead, list(blocks), trail
@property
def name(self):
return None if self.heading is None else self.heading[4:].strip()
def render(self):
out = [] if self.heading is None else [self.heading]
if not self.blocks:
return out + [""] * max(self.lead, self.trail)
out += [""] * self.lead
for i, b in enumerate(self.blocks):
out += ([""] * b.gap if i else []) + b.lines
return out + [""] * self.trail
def append(self, block):
if not self.blocks:
self.lead, self.trail = max(self.lead, 1), max(self.trail, 1)
self.blocks = [Block(block.lines, 0)]
return
prev = self.blocks[-1]
gap = block.gap if block.gap > 0 else (0 if prev.is_bullet and block.is_bullet else 1)
self.blocks.append(Block(block.lines, gap))
def _parse_blocks(lines):
n, i, lead = len(lines), 0, 0
while i < n and _blank(lines[i]):
lead, i = lead + 1, i + 1
blocks, gap = [], 0
while i < n:
start = i
fence = _FENCE.match(lines[i])
if fence and not _indented(lines[i]):
i += 1
while i < n and not lines[i].lstrip().startswith(fence.group(1)):
i += 1
i = min(i + 1, n)
else:
bullet = bool(_BULLET.match(lines[i]))
in_fence = None
i += 1
while i < n:
line = lines[i]
if in_fence:
if line.lstrip().startswith(in_fence):
in_fence = None
i += 1
continue
if _blank(line):
if not bullet:
break
j = i
while j < n and _blank(lines[j]):
j += 1
if j < n and _indented(lines[j]): # nested content of the bullet
i = j
continue
break
if _BULLET.match(line):
break
f = _FENCE.match(line)
if f:
if not _indented(line):
break
in_fence = f.group(1)
i += 1
blocks.append(Block(lines[start:i], gap))
gap = 0
while i < n and _blank(lines[i]):
gap, i = gap + 1, i + 1
return lead, blocks, (gap if blocks else 0)
def _section_version(heading):
m = re.match(r"^## \[([^\]]+)\]", heading) or re.match(r"^## (\S+)", heading)
if not m:
return None
if m.group(1).lower() == "unreleased":
return UNRELEASED
return Version.try_parse(m.group(1))
class Section:
__slots__ = ("heading", "subs", "version")
def __init__(self, heading, subs):
self.heading, self.subs = heading, subs
self.version = _section_version(heading)
@property
def preamble(self):
return self.subs[0]
def items(self):
return [(s.name, b) for s in self.subs for b in s.blocks]
def keys(self):
return {(name, b.text) for name, b in self.items()}
def sub(self, name, create=False):
for s in self.subs:
if s.name == name:
return s
if not create:
return None
s = Sub(f"### {name}", 1, [], 1)
self.subs[-1].trail = max(self.subs[-1].trail, 1)
self.subs.append(s)
return s
def key_counts(self):
return collections.Counter((name, b.text) for name, b in self.items())
def remove(self, counts):
"""Drop blocks matching `counts` ({key: n}), at most n occurrences of
each key, oldest (first) first: a shipped block is removed once, so an
identical entry added again later stays. Drops emptied subsections."""
left, removed = collections.Counter(counts), collections.Counter()
for s in self.subs:
keep = []
for b in s.blocks:
if left[(s.name, b.text)] > 0:
left[(s.name, b.text)] -= 1
removed[(s.name, b.text)] += 1
else:
keep.append(b)
if len(keep) != len(s.blocks):
s.blocks = keep
if not keep and s.heading is None:
s.lead, s.trail = 1, 0
self.subs = [self.subs[0]] + [s for s in self.subs[1:] if s.blocks]
return removed
def append_items(self, items):
"""Append (subsection, block) pairs at the end of their subsections.
Nothing is de-duplicated (multiset semantics, like remove()): an
entry written twice is two entries. Returns how many were added."""
added = 0
for name, block in items:
self.sub(name, create=True).append(block)
added += 1
# The preamble needs a blank line before a following subsection.
if self.preamble.blocks and len(self.subs) > 1:
self.preamble.trail = max(self.preamble.trail, 1)
return added
def return_items(self, items):
"""Put blocks of an unshipped rc back: (subsection, block) pairs, in
their original order, go *before* the blocks already in each
subsection (they were written earlier than anything there now);
subsections that do not exist yet are created. Nothing is
de-duplicated (multiset semantics, like remove()). Returns the count."""
groups = {}
for name, block in items:
groups.setdefault(name, []).append(block)
for name, blocks in groups.items():
sub = self.sub(name, create=True)
old, sub.blocks = sub.blocks, []
for b in [Block(b.lines, 0) for b in blocks] + [
Block(b.lines, b.gap if i else 0) for i, b in enumerate(old)]:
sub.append(b)
if self.preamble.blocks and len(self.subs) > 1:
self.preamble.trail = max(self.preamble.trail, 1)
return sum(len(b) for b in groups.values())
def canonicalize(self):
"""Order subsections canonically (SUBSECTION_ORDER); block order
within a subsection is kept."""
self.subs = [self.subs[0]] + sorted(self.subs[1:], key=lambda s: subsection_rank(s.name))
for s in self.subs[1:-1]:
s.trail = max(s.trail, 1)
if self.preamble.blocks and len(self.subs) > 1:
self.preamble.trail = max(self.preamble.trail, 1)
def render(self):
out = [self.heading]
for s in self.subs:
out += s.render()
return out
def _parse_section(heading, body):
groups, cur = [[None, []]], None
in_fence = None
for line in body:
f = _FENCE.match(line)
if in_fence:
if f and line.lstrip().startswith(in_fence):
in_fence = None
elif f:
in_fence = f.group(1)
elif line.startswith("### "):
groups.append([line, []])
continue
groups[-1][1].append(line)
subs = []
for h, lines in groups:
lead, blocks, trail = _parse_blocks(lines)
subs.append(Sub(h, lead, blocks, trail))
return Section(heading, subs)
class Changelog:
def __init__(self, header, sections, ends_nl, eol="\n"):
self.header, self.sections, self.ends_nl, self.eol = header, sections, ends_nl, eol
@classmethod
def parse(cls, text):
# A CRLF file (a Windows checkout: CHANGELOG.md is plain `text` in
# .gitattributes) is parsed as LF and rendered back as CRLF, so every
# generated line gets the file's own line ending.
eol = "\r\n" if _is_crlf(text) else "\n"
if eol == "\r\n":
text = text.replace("\r\n", "\n")
ends_nl = text.endswith("\n")
lines = text.split("\n")
if ends_nl:
lines = lines[:-1]
header, raw, in_fence = [], [], None
for line in lines:
f = _FENCE.match(line)
if in_fence:
if f and line.lstrip().startswith(in_fence):
in_fence = None
elif f:
in_fence = f.group(1)
elif line.startswith("## "):
raw.append((line, []))
continue
(raw[-1][1] if raw else header).append(line)
return cls(header, [_parse_section(h, body) for h, body in raw], ends_nl, eol)
def render(self):
out = list(self.header)
for s in self.sections:
out += s.render()
return self.eol.join(out) + (self.eol if self.ends_nl else "")
def section(self, version):
for s in self.sections:
if s.version == version:
return s
return None
def unreleased(self):
found = [s for s in self.sections if s.version == UNRELEASED]
if len(found) != 1:
raise ReleaseError(f"CHANGELOG.md must have exactly one '## [Unreleased]' section, found {len(found)}")
return found[0]
def versioned(self):
return [s for s in self.sections if isinstance(s.version, Version)]
def newest_stable(self):
stables = [s.version for s in self.versioned() if not s.version.is_rc]
return max(stables) if stables else None
def insert(self, section):
"""Insert in descending precedence below [Unreleased]."""
if self.section(section.version) is not None:
raise ReleaseError(f"CHANGELOG.md already has a [{section.version}] section")
idx = len(self.sections)
for i, s in enumerate(self.sections):
if isinstance(s.version, Version) and s.version < section.version:
idx = i
break
else:
# Older than everything: after the last versioned section (or [Unreleased]).
for i, s in enumerate(self.sections):
if isinstance(s.version, Version) or s.version == UNRELEASED:
idx = i + 1
unrel = self.sections.index(self.unreleased())
idx = max(idx, unrel + 1)
if idx > 0:
self.sections[idx - 1].subs[-1].trail = max(self.sections[idx - 1].subs[-1].trail, 1)
if idx < len(self.sections):
section.subs[-1].trail = max(section.subs[-1].trail, 1)
self.sections.insert(idx, section)
return idx
def drop(self, section):
self.sections.remove(section)
def _heading(version, date):
if not re.match(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$", date or ""):
raise ReleaseError(f"'{date}' is not a YYYY-MM-DD date")
return f"## [{version}] — {date}"
# Canonical `###` order of a cut section: breaking headings first, then Keep a
# Changelog's order, then anything else; ties by name. A cut always uses it,
# because the order [Unreleased] happens to have on main depends on whether
# release-sync PRs merged (on an unsynced main, already-shipped subsections
# keep their old positions), and the cut must not.
SUBSECTION_ORDER = ("added", "changed", "deprecated", "removed", "fixed", "security")
def subsection_rank(name):
low = (name or "").lower()
kac = next((i for i, k in enumerate(SUBSECTION_ORDER) if low.startswith(k)), len(SUBSECTION_ORDER))
return (0 if "breaking" in low else 1, kac, low, name or "")
def heading_level(name):
name = name.lower()
if "breaking" in name or name.startswith("removed"):
return "major"
if name.startswith(("added", "changed", "deprecated")):
return "minor"
return "patch"
def bump_level(section):
"""major / minor / patch from the subsection names that hold entries."""
order = ("patch", "minor", "major")
levels = [heading_level(s.name) for s in section.subs[1:] if s.blocks]
return max(levels, key=order.index, default="patch")
def sync_main_changelog(text, tags):
"""See the module docstring. `tags`: a TagSource-like object."""
cl = Changelog.parse(text)
unrel = cl.unreleased()
floor = Version.parse(TRAIN_FLOOR)
versions = tags.versions
stables = sorted(v for v in versions if not v.is_rc)
train = [v for v in versions if v > floor]
report = {"inserted": [], "folded": [], "abandoned": [], "removedFromUnreleased": 0,
"returnedToUnreleased": 0}
def tag_section(v):
sec = Changelog.parse(tags.changelog(v)).section(v)
if sec is None:
raise ReleaseError(f"tag v{v} has no [{v}] section in its CHANGELOG.md")
return copy.deepcopy(sec)
def owner(v):
return next((s for s in stables if s >= v.core), None)
# 1. Shipped-block budget per stable: [S]'s blocks (a multiset; the fold
# keeps every occurrence, so [S] is the sum of its folded rcs).
budget, new_stables = {}, []
for v in (v for v in train if not v.is_rc):
if cl.section(v) is None:
sec = tag_section(v)
cl.insert(sec)
budget[v] = sec.key_counts()
new_stables.append(v)
report["inserted"].append(str(v))
# 2. Every rc section whose core has shipped is replaced by what did not
# ship: its blocks (text from its tag, the bytes that were cut) are
# charged against [S]'s budget, oldest rc first, and the rest return.
# A folded rc returns nothing; a later rc abandoned at promotion, or
# a train rc cut beside a hotfix of the same core, returns exactly
# what [S] lacks.
returned = []
for sec in sorted((s for s in cl.versioned() if s.version.is_rc), key=lambda s: s.version):
stable = owner(sec.version)
if stable is None:
continue
if stable not in budget:
shipped = cl.section(stable)
budget[stable] = shipped.key_counts() if shipped else collections.Counter()
left = budget[stable]
source = tag_section(sec.version) if sec.version in versions else sec
back = []
for name, block in source.items():
if left[(name, block.text)] > 0:
left[(name, block.text)] -= 1
else:
back.append((name, block))
returned += back
report["abandoned" if back else "folded"].append(str(sec.version))
cl.drop(sec)
# 3. What the rc sections on main did not cover shipped from blocks that
# are still in [Unreleased] (an unsynced main): remove those, oldest
# occurrence first. Charging the rc sections first means a newer
# identical entry in [Unreleased] (which did not ship) is never
# taken in place of an rc's copy, so the result, block order
# included, does not depend on which release-sync PRs merged.
for v in new_stables:
report["removedFromUnreleased"] += sum(unrel.remove(budget[v]).values())
report["returnedToUnreleased"] += unrel.return_items(returned)
# 4. Pending rc tags.
for v in (v for v in train if v.is_rc and owner(v) is None):
if cl.section(v) is None:
sec = tag_section(v)
cl.insert(sec)
report["removedFromUnreleased"] += sum(unrel.remove(sec.key_counts()).values())
report["inserted"].append(str(v))
return cl.render(), report
def roll_unreleased(text, version, date):
"""[Unreleased] -> `## [V] — date`, leaving an empty [Unreleased] above."""
version = Version.parse(str(version))
cl = Changelog.parse(text)
unrel = cl.unreleased()
if not unrel.items():
raise ReleaseError("CHANGELOG.md's [Unreleased] section is empty — nothing to release")
newer = [s.version for s in cl.versioned() if s.version >= version]
if newer:
raise ReleaseError(f"CHANGELOG.md already has a section >= {version}: [{max(newer)}]")
sec = Section(_heading(version, date), unrel.subs)
sec.canonicalize()
unrel.subs = [Sub(None, 1, [], 0)]
cl.insert(sec)
return cl.render()
def cut_changelog(text, version, date, tags=None):
if tags is not None:
text, _ = sync_main_changelog(text, tags)
return roll_unreleased(text, version, date)
def promote_changelog(text, rc, date):
"""Fold the rc sections into `## [core(rc)] — date` (module docstring)."""
rc = Version.parse(str(rc))
if not rc.is_rc:
raise ReleaseError(f"promote needs an rc version, got {rc}")
cl = Changelog.parse(text)
if cl.section(rc.core) is not None:
raise ReleaseError(f"CHANGELOG.md already has a [{rc.core}] section")
floor = cl.newest_stable()
rcs = [s for s in cl.versioned() if s.version.is_rc]
fold = sorted((s for s in rcs if s.version <= rc and (floor is None or s.version > floor)),
key=lambda s: s.version)
if not fold or fold[-1].version != rc:
raise ReleaseError(f"CHANGELOG.md has no [{rc}] section to promote")
merged = copy.deepcopy(fold[0])
merged.heading, merged.version = _heading(rc.core, date), rc.core
for s in fold[1:]:
merged.append_items(s.items())
unrel = cl.unreleased()
later = sorted((s for s in rcs if s.version.core == rc.core and s.version > rc),
key=lambda s: s.version)
# [core] is exactly the multiset sum of the folded rcs, so none of a
# later rc's blocks shipped through it: all of them return (the same
# count sync-main step 2 charges, which leaves no budget for later rcs).
unrel.return_items([(n, b) for s in later for n, b in s.items()])
for s in later:
cl.drop(s)
newest = fold[-1]
merged.subs[-1].trail = newest.subs[-1].trail
cl.sections[cl.sections.index(newest)] = merged
for s in fold[:-1]:
cl.drop(s)
return cl.render()
def check_section(text, version):
version = Version.parse(str(version))
cl = Changelog.parse(text)
sec = cl.section(version)
if sec is None:
raise ReleaseError(f"CHANGELOG.md has no '## [{version}]' section")
if not sec.items():
raise ReleaseError(f"CHANGELOG.md's [{version}] section is empty — a release needs written notes")
if not version.is_rc:
left = [str(s.version) for s in cl.versioned() if s.version.is_rc and s.version.core == version]
if left:
raise ReleaseError(f"CHANGELOG.md still has unfolded rc sections for {version}: {left}")
return len(sec.items())
# ── version selection ───────────────────────────────────────────────────────
def select_version(tag_versions, branch_versions, unreleased_section):
"""The next rc version (module docstring, "Version rules")."""
tags = sorted(tag_versions)
stables = [v for v in tags if not v.is_rc]