Improve app structure with __main__ file; Add argparse; Improve dead link detection

Author: Raymi306

Makefile

@@ -1,6 +1,6 @@
 .PHONY: default
-default:
-	@echo "Read the Makefile..."
+default: build
+
 .PHONY: clean
 clean:
 	rm -rf dist
@@ -12,10 +12,10 @@ init_dist_dir:
 	mkdir -p dist
 .PHONY: basic_build
 basic_build: clean init_dist_dir
-	python -m app.build
+	python -m app
 .PHONY: basic_build_new_internal_links
 basic_build_new_internal_links: clean init_dist_dir
-	python -m app.build --new-internal-links
+	python -m app --new-internal-links
 .PHONY: pygmentize_css
 pygmentize_css:
 	pygmentize -S solarized-light -f html -a .codehilite > dist/styles.css

app/build.py app/__main__.pyapp/build.py renamed to app/__main__.py

@@ -1,4 +1,5 @@
 """Module for building and build-time QA of the python tutorial website"""
+import argparse
 from os import makedirs
 from pathlib import Path
 import sys
@@ -12,15 +13,24 @@
 from app.templates import TEMPLATES, BASE_HTML
 from app.templates.codeblocks import codeblocks
 from app.templates.links import links
-from app.qa import LinkTester, CodeblocksTester, UnresolvedTemplateVariablesTester, BuildTestFailure
+from app.qa import (
+    LinkTester, CodeblocksTester, UnresolvedTemplateVariablesTester, BuildTestFailure
+)
 
+parser = argparse.ArgumentParser()
+parser.add_argument('--new-internal-links')
+parser.add_argument(
+    '--dry-run',
+    help="""
+    Do not write any files or directories.
+    Still perform network requests for link checks
+    """
+)
 
-def run_build_checks(env, ctx):
+def run_build_checks(env, ctx, args):
     """QA for build process"""
-    args = sys.argv[1:]
-    ignore_internal = '--new-internal-links' in args
     testers = (
-            LinkTester(ignore_internal_links=ignore_internal),
+            LinkTester(ignore_internal_links=args.new_internal_links),
             CodeblocksTester(),
             UnresolvedTemplateVariablesTester(env, ctx),
             )
@@ -60,13 +70,11 @@ def render_templates(env, ctx):
 
 def write_render(render, path):
     """Write final render to appropriate path"""
-    args = sys.argv[1:]
-    if '--dry-run' not in args:
-        with open(path, 'w', encoding='utf-8') as f:
-            f.write(render)
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write(render)
 
 
-def build():
+def main(args):
     """
     Perform all of the actions necessary to output the python tutorial webpages
     """
@@ -76,11 +84,13 @@ def build():
             undefined=DebugUndefined,
             )
     ctx = {**codeblocks, **links}
-    run_build_checks(env, ctx)
-    makedirs('dist', exist_ok=True)
-    for render, path in render_templates(env, ctx):
-        write_render(render, path)
+    run_build_checks(env, ctx, args)
+    if not args.dry_run:
+        makedirs('dist', exist_ok=True)
+        for render, path in render_templates(env, ctx):
+            write_render(render, path)
 
 
 if __name__ == '__main__':
-    build()
+    args_ = parser.parse_args()
+    main(args_)

app/qa.py

@@ -80,6 +80,7 @@ async def fetch(session, url, lock):
                 return await session.get(url, allow_redirects=False, timeout=5)
             except asyncio.exceptions.TimeoutError:
                 print(f"Took too long trying to fetch {url}")
+                return None
 
     async def get_failures(self, links):
         """
@@ -91,7 +92,7 @@ async def get_failures(self, links):
             responses = await asyncio.gather(
                 *[self.fetch(session, url, lock) for url in links]
             )
-            failures = [r for r in responses if r.status != 200]
+            failures = [r for r in responses if r is not None and r.status != 200]
             return failures
 
     def _test(self):