442 lines
14 KiB
Python
Executable File
442 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# SPDX-FileCopyrightText: 2023 Blender Authors
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
|
|
|
"""
|
|
Example use to generate all credits:
|
|
|
|
credits_git_gen.py
|
|
|
|
Example use a custom range:
|
|
|
|
credits_git_gen.py --source=/src/blender --range=SHA1..HEAD
|
|
"""
|
|
__all__ = (
|
|
"main",
|
|
)
|
|
|
|
|
|
# NOTE: this shares the basic structure with `credits_git_gen.py`,
|
|
# however details differ enough for them to be separate scripts.
|
|
# Improvements to this script may apply there too.
|
|
|
|
import argparse
|
|
import html
|
|
import io
|
|
import multiprocessing
|
|
import os
|
|
import sys
|
|
import unicodedata
|
|
|
|
from collections.abc import (
|
|
Iterable,
|
|
)
|
|
|
|
from git_log import (
|
|
GitCommitIter,
|
|
GitCommit,
|
|
)
|
|
|
|
import git_data_canonical_authors
|
|
import git_data_sha1_override_authors
|
|
|
|
IS_ATTY = sys.stdout.isatty()
|
|
|
|
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
|
|
|
|
SOURCE_DIR = os.path.normpath(os.path.join(BASE_DIR, "..", ".."))
|
|
|
|
author_table = git_data_canonical_authors.canonical_author_map()
|
|
author_override_table = git_data_sha1_override_authors.sha1_authors_map()
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Multi-Processing
|
|
|
|
def process_commits_for_map(commits: Iterable[GitCommit]) -> "Credits":
|
|
result = Credits()
|
|
for c in commits:
|
|
result.process_commit(c)
|
|
return result
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Progress Display
|
|
|
|
def value_as_percentage(value_partial: int, value_final: int) -> str:
|
|
percent = 0.0 if (value_final == 0) else (value_partial / value_final)
|
|
return "{:-6.2f}%".format(percent * 100)
|
|
|
|
|
|
if IS_ATTY:
|
|
def progress_output(value_partial: int, value_final: int, info: str) -> None:
|
|
sys.stdout.write("\r\033[K[{:s}]: {:s}".format(value_as_percentage(value_partial, value_final), info))
|
|
else:
|
|
def progress_output(value_partial: int, value_final: int, info: str) -> None:
|
|
sys.stdout.write("[{:s}]: {:s}\n".format(value_as_percentage(value_partial, value_final), info))
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Class for generating credits
|
|
|
|
class CreditUser:
|
|
__slots__ = (
|
|
"commit_total",
|
|
"year_min",
|
|
"year_max",
|
|
)
|
|
|
|
def __init__(self) -> None:
|
|
self.commit_total = 0
|
|
self.year_min = 0
|
|
self.year_max = 0
|
|
|
|
|
|
class Credits:
|
|
__slots__ = (
|
|
"users",
|
|
# Use for progress, simply the number of times `process_commit` has been called.
|
|
"process_commits_count",
|
|
)
|
|
|
|
def __init__(self) -> None:
|
|
self.users: dict[str, CreditUser] = {}
|
|
self.process_commits_count = 0
|
|
|
|
@classmethod
|
|
def commit_authors_get(cls, c: GitCommit) -> list[str]:
|
|
if (authors_overwrite := author_override_table.get(c.sha1, None)) is not None:
|
|
# Ignore git commit info for these having an entry in `author_override_table`.
|
|
return [author_table.get(author, author) for author in authors_overwrite]
|
|
|
|
authors = [c.author] + c.co_authors
|
|
# Normalize author string into canonical form, prevents duplicate credit users
|
|
authors = [unicodedata.normalize('NFC', author) for author in authors]
|
|
return [author_table.get(author, author) for author in authors]
|
|
|
|
@classmethod
|
|
def is_credit_commit_valid(cls, c: GitCommit) -> bool:
|
|
ignore_dir = (
|
|
b"blender/extern/",
|
|
b"blender/intern/opennl/",
|
|
)
|
|
|
|
if not any(f for f in c.files if not f.startswith(ignore_dir)):
|
|
return False
|
|
|
|
return True
|
|
|
|
def merge(self, other: "Credits") -> None:
|
|
"""
|
|
Merge other Credits into this, clearing the other.
|
|
"""
|
|
for user_key, user_other in other.users.items():
|
|
user = self.users.get(user_key)
|
|
if user is None:
|
|
# Consume the user.
|
|
self.users[user_key] = user_other
|
|
else:
|
|
user.commit_total += user_other.commit_total
|
|
user.year_min = min(user.year_min, user_other.year_min)
|
|
user.year_max = max(user.year_max, user_other.year_max)
|
|
other.users.clear()
|
|
|
|
def process_commit(self, c: GitCommit) -> None:
|
|
self.process_commits_count += 1
|
|
|
|
if not self.is_credit_commit_valid(c):
|
|
return
|
|
|
|
authors = self.commit_authors_get(c)
|
|
year = c.date.year
|
|
for author in authors:
|
|
cu = self.users.get(author)
|
|
if cu is None:
|
|
cu = self.users[author] = CreditUser()
|
|
cu.year_min = year
|
|
cu.year_max = year
|
|
|
|
cu.commit_total += 1
|
|
cu.year_min = min(cu.year_min, year)
|
|
cu.year_max = max(cu.year_max, year)
|
|
|
|
def _process_multiprocessing(self, commit_iter: Iterable[GitCommit], *, jobs: int) -> None:
|
|
print("Collecting commits...")
|
|
# NOTE(@ideasman42): that the chunk size doesn't have as much impact on
|
|
# performance as you might expect, values between 16 and 1024 seem reasonable.
|
|
# Although higher values tend to bottleneck as the process finishes.
|
|
chunk_size = 256
|
|
chunk_list = []
|
|
chunk = []
|
|
for c in commit_iter:
|
|
chunk.append(c)
|
|
if len(chunk) >= chunk_size:
|
|
chunk_list.append(chunk)
|
|
chunk = []
|
|
if chunk:
|
|
chunk_list.append(chunk)
|
|
|
|
commit_count_total = (max(len(chunk_list) - 1, 0) * chunk_size) + len(chunk)
|
|
|
|
print("Found {:,d} commits, processing...".format(commit_count_total))
|
|
commit_count_handled = 0
|
|
with multiprocessing.Pool(processes=jobs) as pool:
|
|
for result in pool.imap_unordered(process_commits_for_map, chunk_list):
|
|
commit_count_handled += result.process_commits_count
|
|
progress_output(
|
|
commit_count_handled,
|
|
commit_count_total,
|
|
"{:,d} / {:,d} commits".format(commit_count_handled, commit_count_total),
|
|
)
|
|
self.merge(result)
|
|
|
|
if IS_ATTY:
|
|
print("") # Was printing on one-line, move to next.
|
|
|
|
def process(self, commit_iter: Iterable[GitCommit], *, jobs: int) -> None:
|
|
if jobs > 1:
|
|
self._process_multiprocessing(commit_iter, jobs=jobs)
|
|
return
|
|
|
|
# Simple single process operation.
|
|
for i, c in enumerate(commit_iter):
|
|
self.process_commit(c)
|
|
if not (i % 100):
|
|
print(i)
|
|
|
|
def write_object(
|
|
self,
|
|
fh: io.TextIOWrapper,
|
|
*,
|
|
is_main_credits: bool = True,
|
|
contrib_companies: tuple[str, ...] = (),
|
|
sort: str = "name",
|
|
use_email: bool = False,
|
|
) -> None:
|
|
commit_word = "commit", "commits"
|
|
from unicodedata import normalize
|
|
# Normalize using decomposed strings for sorting.
|
|
# That makes letters with diacritics grouped with the undiacritized variant.
|
|
# Collation is not perfect, but better than having non-English letters at the very bottom of the list.
|
|
|
|
if sort == "commit":
|
|
sorted_authors = dict(sorted(
|
|
self.users.items(),
|
|
key=lambda item: (item[1].commit_total, normalize('NFD', item[0]).casefold()),
|
|
))
|
|
else:
|
|
sorted_authors = dict(sorted(
|
|
self.users.items(),
|
|
key=lambda item: (normalize('NFD', item[0]).casefold(), item[1].commit_total),
|
|
))
|
|
|
|
fh.write("<h3>Individual Contributors</h3>\n\n")
|
|
for author, cu in sorted_authors.items():
|
|
fh.write("{:s}, {:,d} {:s} {:s}<br />\n".format(
|
|
html.escape(author if use_email else author.split("<", 1)[0].rstrip()),
|
|
cu.commit_total,
|
|
commit_word[cu.commit_total > 1],
|
|
(
|
|
"" if not is_main_credits else (
|
|
"- {:d}".format(cu.year_min) if cu.year_min == cu.year_max else
|
|
"({:d} - {:d})".format(cu.year_min, cu.year_max)
|
|
)
|
|
),
|
|
))
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Companies, hard coded
|
|
if is_main_credits:
|
|
fh.write("<h3>Contributions from Companies & Organizations</h3>\n")
|
|
fh.write("<p>\n")
|
|
for line in contrib_companies:
|
|
fh.write("{:s}<br />\n".format(line))
|
|
fh.write("</p>\n")
|
|
|
|
import datetime
|
|
now = datetime.datetime.now()
|
|
fn = os.path.basename(__file__)
|
|
fh.write(
|
|
"<p><center><i>Generated by '{:s}' {:d}/{:02d}/{:02d}</i></center></p>\n".format(
|
|
fn, now.year, now.month, now.day,
|
|
))
|
|
|
|
def write(
|
|
self,
|
|
filepath: str,
|
|
*,
|
|
is_main_credits: bool = True,
|
|
contrib_companies: tuple[str, ...] = (),
|
|
sort: str = "name",
|
|
use_email: bool = False,
|
|
) -> None:
|
|
with open(filepath, 'w', encoding="ascii", errors='xmlcharrefreplace') as fh:
|
|
self.write_object(
|
|
fh,
|
|
is_main_credits=is_main_credits,
|
|
contrib_companies=contrib_companies,
|
|
sort=sort,
|
|
use_email=use_email,
|
|
)
|
|
|
|
|
|
def argparse_create() -> argparse.ArgumentParser:
|
|
|
|
# When `--help` or no arguments are given, print this help.
|
|
usage_text = "Review revisions."
|
|
|
|
epilog = "This script is used to generate credits"
|
|
|
|
parser = argparse.ArgumentParser(description=usage_text, epilog=epilog)
|
|
|
|
parser.add_argument(
|
|
"--source",
|
|
dest="source_dir",
|
|
metavar='PATH',
|
|
required=False,
|
|
default=SOURCE_DIR,
|
|
help="Path to git repository",
|
|
)
|
|
parser.add_argument(
|
|
"--range",
|
|
dest="range_sha1",
|
|
metavar='SHA1_RANGE',
|
|
required=False,
|
|
default="HEAD",
|
|
help="Range to use, eg: 169c95b8..HEAD or HEAD for all history.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--sort",
|
|
dest="sort",
|
|
metavar='METHOD',
|
|
required=False,
|
|
help="Sort credits by 'name' (default) or 'commit'",
|
|
)
|
|
# Don't include email addresses, they're useful for identifying developers but better not include in credits.
|
|
# Even though it's publicly available, developers may not want this so easily accessible on the credits page.
|
|
parser.add_argument(
|
|
"--use-email",
|
|
dest="use_email",
|
|
required=False,
|
|
action='store_true',
|
|
help="Include the email address in the credits (useful for debugging/investigating issues)",
|
|
)
|
|
parser.add_argument(
|
|
"--jobs",
|
|
dest="jobs",
|
|
type=int,
|
|
default=0,
|
|
help=(
|
|
"The number of processes to use. "
|
|
"Defaults to zero which detects the available cores, 1 is single threaded (useful for debugging)."
|
|
),
|
|
required=False,
|
|
)
|
|
|
|
return parser
|
|
|
|
|
|
def parse_organizations(authors_file: str) -> tuple[str, ...] | None:
|
|
"""
|
|
Parse the organizations section from the AUTHORS file.
|
|
Return the sorted organizations or None if they could not be found.
|
|
"""
|
|
orgs = []
|
|
section_begin = "# BEGIN organizations section"
|
|
section_end = "# END organizations section"
|
|
in_section = False
|
|
found_end = False
|
|
|
|
with open(authors_file, "r", encoding="utf-8") as fh:
|
|
for line in fh:
|
|
if line.startswith(section_begin):
|
|
in_section = True
|
|
continue
|
|
if line.startswith(section_end):
|
|
found_end = True
|
|
break
|
|
|
|
if in_section and line and not line.startswith("#"):
|
|
# Extract organization name before `<`.
|
|
org_name = line.split("<")[0].strip()
|
|
# Expected result: `<b>Blender Foundation</b>`.
|
|
orgs.append("<b>{:s}</b>".format(html.escape(org_name)))
|
|
|
|
if not in_section:
|
|
print("Error: Could not find \"{:s}\" in AUTHORS file: {:s}".format(section_begin, authors_file))
|
|
return None
|
|
if not found_end:
|
|
print("Error: Could not find \"{:s}\" in AUTHORS file: {:s}".format(section_end, authors_file))
|
|
return None
|
|
|
|
sorted_unique_organizations = tuple(sorted(set(orgs), key=str.casefold))
|
|
return sorted_unique_organizations
|
|
|
|
|
|
def main() -> int:
|
|
|
|
# ----------
|
|
# Parse Args
|
|
|
|
args = argparse_create().parse_args()
|
|
|
|
# Get organizations in HTML `<b>` tags.
|
|
authors_file = os.path.join(args.source_dir, "AUTHORS")
|
|
contrib_companies = parse_organizations(authors_file)
|
|
if contrib_companies is None:
|
|
return 1
|
|
|
|
credits = Credits()
|
|
# commit_range = "HEAD~10..HEAD"
|
|
# commit_range = "blender-v2.81-release..blender-v2.82-release"
|
|
# commit_range = "blender-v2.82-release"
|
|
commit_range = args.range_sha1
|
|
sort = args.sort
|
|
jobs = args.jobs
|
|
if jobs <= 0:
|
|
# Clamp the value, higher values give errors with too many open files.
|
|
# Allow users to manually pass very high values in as they might want to tweak system limits themselves.
|
|
jobs = min(multiprocessing.cpu_count(), 400)
|
|
|
|
credits.process(GitCommitIter(args.source_dir, commit_range), jobs=jobs)
|
|
|
|
# Report when a single name has multiple entries,
|
|
# as they may need deduplicating in git_data_canonical_authors.py.
|
|
unique_author_names = {}
|
|
unique_author_emails = {}
|
|
for author in credits.users:
|
|
author_name, author_email = author.rsplit("<", 1)
|
|
author_name = author_name.rstrip()
|
|
author_email = "<" + author_email
|
|
unique_author_name = unique_author_names.setdefault(author_name, [])
|
|
unique_author_name.append(author_email)
|
|
|
|
unique_author_email = unique_author_emails.setdefault(author_email, [])
|
|
unique_author_email.append(author_name)
|
|
for name, emails in sorted(unique_author_names.items()):
|
|
if len(emails) > 1:
|
|
print(f"More than one entry for {name}:")
|
|
for email in sorted(emails):
|
|
print(f' "{name} {email}",')
|
|
for email, names in sorted(unique_author_emails.items()):
|
|
if len(names) > 1:
|
|
print(f"More than one entry for {email}:")
|
|
for name in sorted(names):
|
|
print(f' "{name} {email}",')
|
|
|
|
credits.write(
|
|
"credits.html",
|
|
is_main_credits=True,
|
|
contrib_companies=contrib_companies,
|
|
sort=sort,
|
|
use_email=args.use_email,
|
|
)
|
|
print("Written: credits.html")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|