build: improve backend docs autogenerated marker line

Replace custom rem hugo shortcode template with HTML comment. HTML comments are now
allowed in Hugo without enabling unsafe HTML parsing.

Improve the text in the comment: Remove unnecessary quoting, and avoid impression that
make backenddocs has to be run and results committed, since we have a lint check which
will then report error because we want to prevent manual changes in autogenerated sections.

Disable the markdownlint rule line-length on the autogenerated marker line.

Make the autogenerated marker detection a bit more robust.

See #8942 for more details.

bin/check_autogenerated_edits.py

@@ -51,9 +51,9 @@ def find_regions(lines):
     regions = []
     start = None
     for i, line in enumerate(lines, 1):
-        if "rem autogenerated options start" in line:
+        if line.lstrip().startswith("<!-- autogenerated options start "):
             start = i
-        elif "rem autogenerated options stop" in line and start is not None:
+        elif start is not None and line.rstrip().endswith(" autogenerated options stop -->"):
             regions.append((start, i))
             start = None
     return regions

bin/make_backend_docs.py

@@ -9,10 +9,12 @@ import io
 import subprocess
 from pathlib import Path
 
-marker = "{{< rem autogenerated options"
-start = marker + " start"
-stop = marker + " stop"
-end = ">}}"
+begin = "<!-- "
+end = " -->"
+marker = "autogenerated options"
+line_marker_start_prefix = begin + marker + " start "
+line_marker_stop = begin + marker + " stop" + end
+markdownlint_disable = begin + "markdownlint-disable-line line-length" + end
 
 def find_backends():
     """Return a list of all backends"""
@@ -27,7 +29,7 @@ def output_backend_tool_docs(backend, out, cwd):
     """Output documentation for backend tool to out"""
     out.flush()
     subprocess.call(["./rclone", "--config=/notfound", "backend", "help", backend], stdout=out, stderr=subprocess.DEVNULL)
-    
+
 def alter_doc(backend):
     """Alter the documentation for backend"""
     rclone_bin_dir = Path(sys.path[0]).parent.absolute()
@@ -43,23 +45,23 @@ def alter_doc(backend):
         in_docs = False
         for line in in_file:
             if not in_docs:
-                if start in line:
+                if line.lstrip().startswith(line_marker_start_prefix):
                     in_docs = True
-                    start_full = (start + "\" - DO NOT EDIT - instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs\" " + end + "\n") % (backend, backend)
-                    out_file.write(start_full)
+                    line_marker_start = (line_marker_start_prefix + "- DO NOT EDIT - instead edit fs.RegInfo in backend/%s/%s.go and run make backenddocs to verify" + end) % (backend, backend)
+                    out_file.write(line_marker_start + " " + markdownlint_disable + "\n")
                     output_docs(backend, out_file, rclone_bin_dir)
                     output_backend_tool_docs(backend, out_file, rclone_bin_dir)
-                    out_file.write(stop+" "+end+"\n")
+                    out_file.write(line_marker_stop + "\n")
                     altered = True
             if not in_docs:
                 out_file.write(line)
             if in_docs:
-                if stop in line:
+                if line.strip() == line_marker_stop:
                     in_docs = False
     os.rename(doc_file, doc_file+"~")
     os.rename(new_file, doc_file)
     if not altered:
-        raise ValueError("Didn't find '%s' markers for in %s" % (start, doc_file))
+        raise ValueError("Didn't find '%s' markers in %s" % (line_marker_start_prefix, doc_file))
 
 
 def main(args):

bin/make_manual.py

@@ -152,7 +152,7 @@ def read_doc(doc):
     # Make [...](/links/) absolute
     contents = re.sub(r'\]\((\/.*?\/(#.*)?)\)', r"](https://rclone.org\1)", contents)
     # Add additional links on the front page
-    contents = re.sub(r'\{\{< rem MAINPAGELINK >\}\}', "- [Donate.](https://rclone.org/donate/)", contents)
+    contents = re.sub(r'<!-- MAINPAGELINK -->', "- [Donate.](https://rclone.org/donate/)", contents)
     # Interpret provider shortcode
     # {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
     contents = re.sub(r'\{\{<\s*provider.*?name="(.*?)".*?>\}\}', r"- \1", contents)

bin/make_rc_docs.sh

@@ -15,8 +15,8 @@ fusermount -u -z /tmp/rclone/rc_mount > /dev/null 2>&1 || umount /tmp/rclone/rc_
 
 awk '
     BEGIN       {p=1}
-    /^\{\{< rem autogenerated start/  {print;system("cat /tmp/rclone/z.md");p=0}
-    /^\{\{< rem autogenerated stop/   {p=1}
+    /^<!-- autogenerated start/  {print;system("cat /tmp/rclone/z.md");p=0}
+    /^<!-- autogenerated stop/   {p=1}
     p' docs/content/rc.md > /tmp/rclone/rc.md
 
 mv /tmp/rclone/rc.md docs/content/rc.md