Merge d66a8bf417 into 15ff450680

methods.py

@@ -1441,12 +1441,19 @@ def generate_copyright_header(filename: str) -> str:
 /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
 /**************************************************************************/
 """
-    filename = filename.split("/")[-1].ljust(MARGIN)
+    filename = os.path.basename(filename).ljust(MARGIN)
     if len(filename) > MARGIN:
         print_warning(f'Filename "{filename}" too large for copyright header.')
     return TEMPLATE % filename
 
 
+def generate_header_guard(filename: str, prefix: str = "", suffix: str = "") -> str:
+    base, extension = os.path.basename(filename).split(".", 1)
+    header_guard = "_".join([prefix, base, suffix, extension]).upper()
+    header_guard = re.sub(r"[_\-\.\s]+", "_", header_guard)
+    return header_guard.strip("_")
+
+
 @contextlib.contextmanager
 def generated_wrapper(
     path,  # FIXME: type with `Union[str, Node, List[Node]]` when pytest conflicts are resolved
@@ -1488,15 +1495,7 @@ def generated_wrapper(
     if not guard and (prefix or suffix):
         print_warning(f'Trying to assign header guard prefix/suffix while `guard` is disabled: "{path}".')
 
-    header_guard = ""
+    header_guard = generate_header_guard(path, prefix, suffix) if guard else ""
-    if guard:
-        if prefix:
-            prefix += "_"
-        if suffix:
-            suffix = f"_{suffix}"
-        split = path.split("/")[-1].split(".")
-        header_guard = (f"{prefix}{split[0]}{suffix}.{'.'.join(split[1:])}".upper()
-                .replace(".", "_").replace("-", "_").replace(" ", "_").replace("__", "_"))  # fmt: skip
 
     with open(path, "wt", encoding="utf-8", newline="\n") as file:
         file.write(generate_copyright_header(path))

misc/scripts/copyright_headers.py

@@ -1,96 +1,68 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
+if __name__ != "__main__":
+    raise ImportError(f"{__name__} should not be used as a module.")
+
+import argparse
 import os
 import sys
 
-header = """\
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../"))
-/**************************************************************************/
-/*  $filename                                                             */
-/**************************************************************************/
-/*                         This file is part of:                          */
-/*                             GODOT ENGINE                               */
-/*                        https://godotengine.org                         */
-/**************************************************************************/
-/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
-/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                  */
-/*                                                                        */
-/* Permission is hereby granted, free of charge, to any person obtaining  */
-/* a copy of this software and associated documentation files (the        */
-/* "Software"), to deal in the Software without restriction, including    */
-/* without limitation the rights to use, copy, modify, merge, publish,    */
-/* distribute, sublicense, and/or sell copies of the Software, and to     */
-/* permit persons to whom the Software is furnished to do so, subject to  */
-/* the following conditions:                                              */
-/*                                                                        */
-/* The above copyright notice and this permission notice shall be         */
-/* included in all copies or substantial portions of the Software.        */
-/*                                                                        */
-/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,        */
-/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF     */
-/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
-/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY   */
-/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,   */
-/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE      */
-/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
-/**************************************************************************/
-"""
 
-if len(sys.argv) < 2:
+from methods import generate_copyright_header, print_error, print_warning, toggle_color
-    print("Invalid usage of copyright_headers.py, it should be called with a path to one or multiple files.")
-    sys.exit(1)
 
-for f in sys.argv[1:]:
-    fname = f
 
-    # Handle replacing $filename with actual filename and keep alignment
+def evaluate_header(path: str) -> int:
-    fsingle = os.path.basename(fname.strip())
+    try:
-    rep_fl = "$filename"
+        with open(path, encoding="utf-8", newline="\n") as file:
-    rep_fi = fsingle
+            header = generate_copyright_header(path)
-    len_fl = len(rep_fl)
+            synced = True
-    len_fi = len(rep_fi)
+            for line in header.splitlines(True):
-    # Pad with spaces to keep alignment
+                if line != file.readline():
-    if len_fi < len_fl:
+                    synced = False
-        for x in range(len_fl - len_fi):
+                    break
-            rep_fi += " "
+            if synced:
-    elif len_fl < len_fi:
+                return 0
-        for x in range(len_fi - len_fl):
-            rep_fl += " "
-    if header.find(rep_fl) != -1:
-        text = header.replace(rep_fl, rep_fi)
-    else:
-        text = header.replace("$filename", fsingle)
-    text += "\n"
 
-    # We now have the proper header, so we want to ignore the one in the original file
+            # Header is mangled or missing; remove all empty/commented lines prior to content.
-    # and potentially empty lines and badly formatted lines, while keeping comments that
+            content = header
-    # come after the header, and then keep everything non-header unchanged.
+            file.seek(0)
-    # To do so, we skip empty lines that may be at the top in a first pass.
+            for line in file:
-    # In a second pass, we skip all consecutive comment lines starting with "/*",
+                if line == "\n" or line.startswith("/*"):
-    # then we can append the rest (step 2).
+                    continue
+                content += f"\n{line}"
+                break
+            content += file.read()
 
-    with open(fname.strip(), "r", encoding="utf-8") as fileread:
+        with open(path, "w", encoding="utf-8", newline="\n") as file:
-        line = fileread.readline()
+            file.write(content)
-        header_done = False
 
-        while line.strip() == "" and line != "":  # Skip empty lines at the top
+        print_warning(f'File "{path}" had an improper header. Fixed!')
-            line = fileread.readline()
+        return 1
+    except OSError:
+        print_error(f'Failed to open file "{path}", skipping header check.')
+        return 1
 
-        if line.find("/**********") == -1:  # Godot header starts this way
-            # Maybe starting with a non-Godot comment, abort header magic
-            header_done = True
 
-        while not header_done:  # Handle header now
+def main() -> int:
-            if line.find("/*") != 0:  # No more starting with a comment
+    parser = argparse.ArgumentParser(prog="copyright-headers", description="Ensure files have valid copyright headers.")
-                header_done = True
+    parser.add_argument("files", nargs="+", help="Paths to files for copyright header evaluation.")
-                if line.strip() != "":
+    parser.add_argument("-c", "--color", action="store_true", help="If passed, force colored output.")
-                    text += line
+    args = parser.parse_args()
-            line = fileread.readline()
 
-        while line != "":  # Dump everything until EOF
+    if args.color:
-            text += line
+        toggle_color(True)
-            line = fileread.readline()
 
-    # Write
+    ret = 0
-    with open(fname.strip(), "w", encoding="utf-8", newline="\n") as filewrite:
+    for file in args.files:
-        filewrite.write(text)
+        ret += evaluate_header(file)
+    return ret
+
+
+try:
+    sys.exit(main())
+except KeyboardInterrupt:
+    import signal
+
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
+    os.kill(os.getpid(), signal.SIGINT)

misc/scripts/file_format.py

@@ -1,51 +1,69 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
+if __name__ != "__main__":
+    raise ImportError(f"{__name__} should not be used as a module.")
+
+import argparse
+import os
 import sys
 
-if len(sys.argv) < 2:
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../"))
-    print("Invalid usage of file_format.py, it should be called with a path to one or multiple files.")
-    sys.exit(1)
 
-BOM = b"\xef\xbb\xbf"
+from methods import print_error, print_warning, toggle_color
 
-changed = []
-invalid = []
 
-for file in sys.argv[1:]:
+def evaluate_formatting(path: str) -> int:
     try:
-        with open(file, "rt", encoding="utf-8") as f:
+        with open(path, "rb") as file:
-            original = f.read()
+            raw = file.read()
+
+        if not raw:
+            return 0
+
+        # TODO: Replace hardcoded choices by parsing relevant `.gitattributes`/`.editorconfig`.
+        EOL = "\r\n" if path.endswith((".csproj", ".sln", ".bat")) or path.startswith("misc/msvs") else "\n"
+        WANTS_BOM = path.endswith((".csproj", ".sln"))
+
+        reformat_decode = EOL.join([line.rstrip() for line in raw.decode("utf-8-sig").splitlines()]).rstrip() + EOL
+        reformat_encode = reformat_decode.encode("utf-8-sig" if WANTS_BOM else "utf-8")
+
+        if raw == reformat_encode:
+            return 0
+
+        with open(path, "wb") as file:
+            file.write(reformat_encode)
+
+        print_warning(f'File "{path}" had improper formatting. Fixed!')
+        return 1
+    except OSError:
+        print_error(f'Failed to open file "{path}", skipping format.')
+        return 1
     except UnicodeDecodeError:
-        invalid.append(file)
+        print_error(f'File at "{path}" is not UTF-8, requires manual changes.')
-        continue
+        return 1
 
-    if original == "":
-        continue
 
-    EOL = "\r\n" if file.endswith((".csproj", ".sln", ".bat")) or file.startswith("misc/msvs") else "\n"
+def main() -> int:
-    WANTS_BOM = file.endswith((".csproj", ".sln"))
+    parser = argparse.ArgumentParser(
+        prog="file-format", description="Ensure files have proper formatting (newlines, encoding, etc)."
+    )
+    parser.add_argument("files", nargs="+", help="Paths to files for formatting.")
+    parser.add_argument("-c", "--color", action="store_true", help="If passed, force colored output.")
+    args = parser.parse_args()
 
-    revamp = EOL.join([line.rstrip("\n\r\t ") for line in original.splitlines(True)]).rstrip(EOL) + EOL
+    if args.color:
+        toggle_color(True)
 
-    new_raw = revamp.encode(encoding="utf-8")
+    ret = 0
-    if not WANTS_BOM and new_raw.startswith(BOM):
+    for file in args.files:
-        new_raw = new_raw[len(BOM) :]
+        ret += evaluate_formatting(file)
-    elif WANTS_BOM and not new_raw.startswith(BOM):
+    return ret
-        new_raw = BOM + new_raw
 
-    with open(file, "rb") as f:
-        old_raw = f.read()
 
-    if old_raw != new_raw:
+try:
-        changed.append(file)
+    sys.exit(main())
-        with open(file, "wb") as f:
+except KeyboardInterrupt:
-            f.write(new_raw)
+    import signal
 
-if changed:
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
-    for file in changed:
+    os.kill(os.getpid(), signal.SIGINT)
-        print(f"FIXED: {file}")
-if invalid:
-    for file in invalid:
-        print(f"REQUIRES MANUAL CHANGES: {file}")
-    sys.exit(1)

misc/scripts/header_guards.py

@@ -1,32 +1,39 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
+if __name__ != "__main__":
+    raise ImportError(f"{__name__} should not be used as a module.")
+
+import argparse
+import os
 import sys
-from pathlib import Path
 
-if len(sys.argv) < 2:
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../"))
-    print("Invalid usage of header_guards.py, it should be called with a path to one or multiple files.")
-    sys.exit(1)
 
-changed = []
+from methods import generate_header_guard, print_error, print_warning, toggle_color
-invalid = []
 
-for file in sys.argv[1:]:
+
-    header_start = -1
+def evaluate_header_guards(path: str) -> int:
+    try:
+        with open(path, encoding="utf-8", newline="\n") as file:
+            lines = file.readlines()
+    except OSError:
+        print_error(f'Failed to open file "{path}", skipping header guard check.')
+        return 1
+
+    if not lines:
+        return 0
+
+    header_found = False
     HEADER_CHECK_OFFSET = -1
 
-    with open(file.strip(), "rt", encoding="utf-8", newline="\n") as f:
-        lines = f.readlines()
-
     for idx, line in enumerate(lines):
-        sline = line.strip()
+        sline = line.lstrip()
-
+        if not header_found:
-        if header_start < 0:
+            if not sline:  # Skip empty lines at the top.
-            if sline == "":  # Skip empty lines at the top.
                 continue
 
             if sline.startswith("/**********"):  # Godot header starts this way.
-                header_start = idx
+                header_found = True
             else:
                 HEADER_CHECK_OFFSET = 0  # There is no Godot header.
                 break
@@ -36,51 +43,48 @@ for file in sys.argv[1:]:
                 break
 
     if HEADER_CHECK_OFFSET < 0:
-        invalid.append(file)
+        return 0  # Dummy file.
-        continue
 
     HEADER_BEGIN_OFFSET = HEADER_CHECK_OFFSET + 1
     HEADER_END_OFFSET = len(lines) - 1
 
     if HEADER_BEGIN_OFFSET >= HEADER_END_OFFSET:
-        invalid.append(file)
+        return 0  # Dummy file.
-        continue
 
-    split = file.split("/")  # Already in posix-format.
+    split = path.split("/")  # Already in posix-format.
 
     prefix = ""
     if split[0] == "modules" and split[-1] == "register_types.h":
-        prefix = f"{split[1]}_"  # Name of module.
+        prefix = split[1]  # Name of module.
-    elif split[0] == "platform" and (file.endswith("api/api.h") or "/export/" in file):
+    elif split[0] == "platform" and (path.endswith("api/api.h") or "/export/" in path):
-        prefix = f"{split[1]}_"  # Name of platform.
+        prefix = split[1]  # Name of platform.
-    elif file.startswith("modules/mono/utils") and "mono" not in split[-1]:
+    elif path.startswith("modules/mono/utils") and "mono" not in split[-1]:
-        prefix = "MONO_"
+        prefix = "mono"
-    elif file == "servers/rendering/storage/utilities.h":
+    elif path == "servers/rendering/storage/utilities.h":
-        prefix = "RENDERER_"
+        prefix = "renderer"
 
     suffix = ""
-    if "dummy" in file and "dummy" not in split[-1]:
+    if "dummy" in path and not any("dummy" in x for x in (prefix, split[-1])):
-        suffix = "_DUMMY"
+        suffix = "dummy"
-    elif "gles3" in file and "gles3" not in split[-1]:
+    elif "gles3" in path and not any("gles3" in x for x in (prefix, split[-1])):
-        suffix = "_GLES3"
+        suffix = "gles3"
-    elif "renderer_rd" in file and "rd" not in split[-1]:
+    elif "renderer_rd" in path and not any("rd" in x for x in (prefix, split[-1])):
-        suffix = "_RD"
+        suffix = "rd"
     elif split[-1] == "ustring.h":
-        suffix = "_GODOT"
+        suffix = "godot"
 
-    name = (f"{prefix}{Path(file).stem}{suffix}{Path(file).suffix}".upper()
+    header_guard = generate_header_guard(path, prefix, suffix)
-            .replace(".", "_").replace("-", "_").replace(" ", "_"))  # fmt: skip
 
-    HEADER_CHECK = f"#ifndef {name}\n"
+    HEADER_CHECK = f"#ifndef {header_guard}\n"
-    HEADER_BEGIN = f"#define {name}\n"
+    HEADER_BEGIN = f"#define {header_guard}\n"
-    HEADER_END = f"#endif // {name}\n"
+    HEADER_END = f"#endif // {header_guard}\n"
 
     if (
         lines[HEADER_CHECK_OFFSET] == HEADER_CHECK
         and lines[HEADER_BEGIN_OFFSET] == HEADER_BEGIN
         and lines[HEADER_END_OFFSET] == HEADER_END
     ):
-        continue
+        return 0
 
     # Guards might exist but with the wrong names.
     if (
@@ -91,84 +95,79 @@ for file in sys.argv[1:]:
         lines[HEADER_CHECK_OFFSET] = HEADER_CHECK
         lines[HEADER_BEGIN_OFFSET] = HEADER_BEGIN
         lines[HEADER_END_OFFSET] = HEADER_END
-        with open(file, "wt", encoding="utf-8", newline="\n") as f:
+        try:
-            f.writelines(lines)
+            with open(path, "w", encoding="utf-8", newline="\n") as file:
-        changed.append(file)
+                file.writelines(lines)
-        continue
+            print_warning(f'File "{path}" had improper header guards. Fixed!')
+        except OSError:
+            print_error(f'Failed to open file "{path}", aborting header guard fix.')
+        return 1
 
     header_check = -1
     header_begin = -1
     header_end = -1
     pragma_once = -1
-    objc = False
 
     for idx, line in enumerate(lines):
-        if line.startswith("// #import"):  # Some dummy obj-c files only have commented out import lines.
-            objc = True
-            break
         if not line.startswith("#"):
             continue
         elif line.startswith("#ifndef") and header_check == -1:
             header_check = idx
         elif line.startswith("#define") and header_begin == -1:
             header_begin = idx
-        elif line.startswith("#endif") and header_end == -1:
+        elif line.startswith("#endif"):
             header_end = idx
         elif line.startswith("#pragma once"):
             pragma_once = idx
             break
-        elif line.startswith("#import"):
-            objc = True
-            break
-
-    if objc:
-        continue
 
     if pragma_once != -1:
         lines.pop(pragma_once)
-        lines.insert(HEADER_CHECK_OFFSET, HEADER_CHECK)
+        lines.insert(HEADER_CHECK_OFFSET, HEADER_CHECK + HEADER_BEGIN + "\n")
-        lines.insert(HEADER_BEGIN_OFFSET, HEADER_BEGIN)
+        lines.append("\n" + HEADER_END)
-        lines.append("\n")
+        try:
-        lines.append(HEADER_END)
+            with open(path, "w", encoding="utf-8", newline="\n") as file:
-        with open(file, "wt", encoding="utf-8", newline="\n") as f:
+                file.writelines(lines)
-            f.writelines(lines)
+            print_warning(f'File "{path}" used `#pragma once` instead of header guards. Fixed!')
-        changed.append(file)
+        except OSError:
-        continue
+            print_error(f'Failed to open file "{path}", aborting header guard fix.')
+        return 1
 
     if header_check == -1 and header_begin == -1 and header_end == -1:
-        # Guards simply didn't exist
+        # Guards simply didn't exist.
-        lines.insert(HEADER_CHECK_OFFSET, HEADER_CHECK)
+        lines.insert(HEADER_CHECK_OFFSET, HEADER_CHECK + HEADER_BEGIN + "\n")
-        lines.insert(HEADER_BEGIN_OFFSET, HEADER_BEGIN)
+        lines.append("\n" + HEADER_END)
-        lines.append("\n")
+        try:
-        lines.append(HEADER_END)
+            with open(path, "w", encoding="utf-8", newline="\n") as file:
-        with open(file, "wt", encoding="utf-8", newline="\n") as f:
+                file.writelines(lines)
-            f.writelines(lines)
+            print_warning(f'File "{path}" lacked header guards. Fixed!')
-        changed.append(file)
+        except OSError:
-        continue
+            print_error(f'Failed to open file "{path}", aborting header guard fix.')
+        return 1
 
-    if header_check != -1 and header_begin != -1 and header_end != -1:
+    print_error(f'File "{path}" has invalid header guards, requires manual changes.')
-        # All prepends "found", see if we can salvage this.
+    return 1
-        if header_check == header_begin - 1 and header_begin < header_end:
-            lines.pop(header_check)
-            lines.pop(header_begin - 1)
-            lines.pop(header_end - 2)
-            if lines[header_end - 3] == "\n":
-                lines.pop(header_end - 3)
-            lines.insert(HEADER_CHECK_OFFSET, HEADER_CHECK)
-            lines.insert(HEADER_BEGIN_OFFSET, HEADER_BEGIN)
-            lines.append("\n")
-            lines.append(HEADER_END)
-            with open(file, "wt", encoding="utf-8", newline="\n") as f:
-                f.writelines(lines)
-            changed.append(file)
-            continue
 
-    invalid.append(file)
 
-if changed:
+def main() -> int:
-    for file in changed:
+    parser = argparse.ArgumentParser(prog="header-guards", description="Ensure header files have valid header guards.")
-        print(f"FIXED: {file}")
+    parser.add_argument("files", nargs="+", help="Paths to files for header guard evaluation.")
-if invalid:
+    parser.add_argument("-c", "--color", action="store_true", help="If passed, force colored output.")
-    for file in invalid:
+    args = parser.parse_args()
-        print(f"REQUIRES MANUAL CHANGES: {file}")
+
-    sys.exit(1)
+    if args.color:
+        toggle_color(True)
+
+    ret = 0
+    for file in args.files:
+        ret += evaluate_header_guards(file)
+    return ret
+
+
+try:
+    sys.exit(main())
+except KeyboardInterrupt:
+    import signal
+
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
+    os.kill(os.getpid(), signal.SIGINT)