From 54ef485347cff4f2cd147444c86a102c6e3bb36b Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 23 May 2017 11:42:29 -0700 Subject: [PATCH 1/2] Add script to check for broken deep links Additionally, create a noxfile for running the build and link checker on travis/locally. --- .gitignore | 3 + .travis.yml | 6 +- nox.py | 27 ++++++ requirements.txt | 1 + scripts/linkmonitor/inventory.yaml | 96 +++++++++++++++++++ scripts/linkmonitor/linkmonitor.py | 138 +++++++++++++++++++++++++++ scripts/linkmonitor/requirements.txt | 2 + 7 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 nox.py create mode 100644 requirements.txt create mode 100644 scripts/linkmonitor/inventory.yaml create mode 100644 scripts/linkmonitor/linkmonitor.py create mode 100644 scripts/linkmonitor/requirements.txt diff --git a/.gitignore b/.gitignore index db214a9ae..232183b18 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ build/ .*~ +.nox +*.pyc +__pycache__ diff --git a/.travis.yml b/.travis.yml index 1cb311e14..972bdb771 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python sudo: false python: -- 2.7 -install: pip install sphinx==1.5.6 -script: make SPHINXOPTS="-W" html +- 3.6 +install: pip install --upgrade nox-automation virtualenv +script: nox diff --git a/nox.py b/nox.py new file mode 100644 index 000000000..9e2c95753 --- /dev/null +++ b/nox.py @@ -0,0 +1,27 @@ +# Copyright 2017, PyPA +# The Python Packaging User Guide is licensed under a Creative Commons +# Attribution-ShareAlike license: +# http://creativecommons.org/licenses/by-sa/3.0. + +import os + +import nox + + +@nox.session +def build(session): + session.interpreter = 'python3.6' + session.install('-r', 'requirements.txt') + # Treat warnings as errors. + session.env['SPHINXOPTS'] = '-W' + session.run('make', 'html') + + +@nox.session +def checklinks(session): + if not os.path.exists(os.path.join('build', 'html')): + session.error('HTML output not available, run nox -s build first.') + session.interpreter = 'python3.6' + session.install('-r', 'scripts/linkmonitor/requirements.txt') + session.run( + 'python', 'scripts/linkmonitor/linkmonitor.py', 'check') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..4ee8d0237 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +sphinx==1.5.6 diff --git a/scripts/linkmonitor/inventory.yaml b/scripts/linkmonitor/inventory.yaml new file mode 100644 index 000000000..a359ac9ea --- /dev/null +++ b/scripts/linkmonitor/inventory.yaml @@ -0,0 +1,96 @@ +[science.html, self_hosted_repository.html#hosting-your-own-simple-repository, pip_easy_install.html#pip-vs-easy-install, + glossary.html#term-binary-distribution, specifications.html#recording-installed-distributions, + glossary.html#term-requirement, namespace_packages.html#creating-a-namespace-package, + deployment.html#contents, contribute.html#voice-and-tone, platforms.html, wheel_egg.html, + distributing.html#pure-python-wheels, appveyor.html#setting-up, glossary.html#term-project, + distributing.html#packaging-and-distributing-projects, deployment.html#supporting-multiple-hardware-platforms, + glossary.html#term-release, key_projects.html#warehouse, distributing.html#package-data, + distributing.html#uploading-your-project-to-pypi, distributing.html#packages, key_projects.html#python-packaging-user-guide, + key_projects.html#virtualenv, glossary.html#term-per-project-index, install_requirements_linux.html, + extensions.html, key_projects.html#bandersnatch, glossary.html#term-working-set, + key_projects.html#project-summaries, installing.html#optionally-create-a-virtual-environment, + distributing.html#platform-wheels, key_projects.html#buildout, namespace_packages.html#native-namespace-packages, + installing.html#creating-virtual-environments, installing.html#creating-and-using-virtual-environments, + distributing.html#choosing-a-versioning-scheme, key_projects.html#devpi, glossary.html#term-version-specifier, + distributing.html#universal-wheels, distributing.html#date-based-versioning, specifications.html#package-distribution-metadata, + specifications.html#dependency-specifiers, patching.html#patching-forking, installing.html#installing-from-a-local-src-tree, + distributing.html#scheme-choices, key_projects.html#wheel, key_projects.html#distlib, + science.html#the-conda-cross-platform-package-manager, additional.html#additional-topics, + appveyor.html#testing-with-tox, current.html#packaging-tool-recommendations, distributing.html#wheels, + extensions.html#use-cases, wheel_egg.html#wheel-vs-egg, distributing.html#url, science.html#contents, + distributing.html#install-requires, search.html#fallback, distributing.html#version, + glossary.html#term-requirements-file, installing.html#upgrading-packages, glossary.html#term-python-packaging-authority-pypa, + distributing.html#keywords, distributing.html#manifest-in, key_projects.html#spack, + installing.html#installing-from-local-archives, installing.html#installing-to-the-user-site, + science.html#mac-os-x-installers-and-package-managers, multiple_python_versions.html#automated-testing-and-continuous-integration, + specifications.html#pypa-specifications, plugin_discovery.html#using-namespace-packages, + search.html#search-documentation, search.html#search-results, distributing.html#standards-compliance-for-interoperability, + multiple_python_versions.html#contents, installing.html#installing-packages, distributing.html#license, + extensions.html#alternatives-to-handcoded-wrapper-modules, extensions.html#alternatives-for-low-level-system-access, + requirements.html#contents, extensions.html#building-binary-extensions, specifications.html#package-index-interfaces, + distributing.html#local-version-identifiers, installing.html#source-distributions-vs-wheels, + distributing.html#name, extensions.html#an-overview-of-binary-extensions, key_projects.html#setuptools, + multi_version_install.html, install_requirements_linux.html#fedora, distributing.html#serial-versioning, + quickstart.html#quickstart, contribute.html#conventions-and-mechanics, deployment.html#configuration-management, + single_source_version.html#single-sourcing-the-project-version, installing.html#installing-prereleases, + appveyor.html#appveyor-yml, requirements.html#install-requires, glossary.html#term-module, + key_projects.html#twine, key_projects.html#easy-install, appveyor.html#contents, + plugin_discovery.html#using-naming-convention, science.html#scipy-distributions, + glossary.html#term-system-package, glossary.html#term-virtual-environment, specifications.html#binary-distribution-format, + self_hosted_repository.html, distributing.html#data-files, tutorial.html, multiple_python_versions.html#supporting-multiple-python-versions, + plugin_discovery.html#using-package-metadata, support.html#how-to-get-support, requirements.html#install-requires-vs-requirements-files, + distributing.html#working-in-development-mode, contribute.html#audience, distributing.html#console-scripts, + distributing.html#source-distributions, installing.html#installing-from-other-sources, + distributing.html#hybrid-schemes, appveyor.html#access-to-the-built-wheels, installing.html#install-pip-setuptools-and-wheel, + distributing.html#requirements-for-packaging-and-distributing, glossary.html#term-source-archive, + appveyor.html#external-dependencies, install_requirements_linux.html#centos-rhel, + key_projects.html#ensurepip, installing.html#installing-from-pypi, appveyor.html#additional-notes, + install_requirements_linux.html#debian-ubuntu, key_projects.html#pypa-projects, + patching.html, appveyor.html#supporting-windows-using-appveyor, namespace_packages.html#packaging-namespace-packages, + requirements.html, distributing.html#classifiers, key_projects.html#distutils, current.html#installation-tool-recommendations, + multiple_python_versions.html#what-s-in-which-python, additional.html, distributing.html#entry-points, + distributing.html#setup-args, key_projects.html#venv, specifications.html#core-metadata, + pip_easy_install.html, science.html#spack, science.html#linux-distribution-packages, + installing.html#installing-from-vcs, install_requirements_linux.html#opensuse, key_projects.html#packaging, + install_requirements_linux.html#installing-pip-setuptools-wheel-with-linux-package-managers, + installing.html#requirements-for-installing-packages, glossary.html#term-requirement-specifier, + multi_version_install.html#multi-version-installs, support.html, namespace_packages.html, + contribute.html#style-guide, specifications.html#version-specifiers, specifications.html#platform-compatibility-tags, + distributing.html#readme-rst, namespace_packages.html#pkg-resources-style-namespace-packages, + distributing.html#scripts, contribute.html#contributing-style-guide, distributing.html#pre-release-versioning, + current.html#tool-recommendations, installing.html#use-pip-for-installing, platforms.html#platform-integtation, + glossary.html#term-python-package-index-pypi, contribute.html, glossary.html#term-source-distribution-or-sdist, + extensions.html#setting-up-a-build-environment-on-windows, glossary.html#term-known-good-set-kgs, + mirrors.html#contents, distributing.html#upload-your-distributions, installing.html#installing-requirements, + multiple_python_versions.html#tools-for-single-source-python-packages, deployment.html#windows, + mirrors.html#caching-with-pip, specifications.html#description-content-type, appveyor.html, + contribute.html#purpose, distributing.html#contents, requirements.html#requirements-files, + glossary.html#term-package-index, appveyor.html#background, mirrors.html#pypi-mirrors-and-caches, + key_projects.html#pip, mirrors.html#caching-with-devpi, key_projects.html#hashdist, + distributing.html#your-package, installing.html#installing-from-other-indexes, search.html, + plugin_discovery.html, mirrors.html#complete-mirror-with-bandersnatch, mirrors.html, + distributing.html#author, install_requirements_linux.html#arch-linux, distributing.html, + installing.html, specifications.html, glossary.html#term-setup-py, extensions.html#implementing-binary-extensions, + glossary.html#term-egg, plugin_discovery.html#plugin-creation-and-discovery, glossary.html#glossary, + deployment.html#application-deployment, deployment.html#os-packaging-installers, + glossary.html#term-built-distribution, installing.html#contents, deployment.html#pynsist, + glossary.html#term-wheel, multiple_python_versions.html, contribute.html#contribute-to-this-guide, + science.html#building-from-source, specifications.html#simple-repository-api, deployment.html#overview, + key_projects.html#bento, key_projects.html, appveyor.html#adding-appveyor-support-to-your-project, + distributing.html#packaging-your-project, appveyor.html#support-script, namespace_packages.html#pkgutil-style-namespace-packages, + extensions.html#binary-extensions, key_projects.html#conda, installing.html#installing-setuptools-extras, + distributing.html#configuring-your-project, key_projects.html#pex, key_projects.html#non-pypa-projects, + self_hosted_repository.html#manual-repository, distributing.html#create-an-account, + key_projects.html#projects, search.html#search-progress, science.html#windows-installers, + key_projects.html#standard-library-projects, distributing.html#initial-files, distributing.html#description, + glossary.html#term-distribution-package, appveyor.html#support-scripts, specifications.html#declaring-build-system-dependencies, + deployment.html, extensions.html#publishing-binary-extensions, glossary.html#term-pure-module, + glossary.html#term-import-package, appveyor.html#automatically-uploading-wheels, + single_source_version.html#single-sourcing-the-version, science.html#numpy-and-the-science-stack, + science.html#installing-scientific-packages, specifications.html#source-distribution-format, + distributing.html#semantic-versioning-preferred, distributing.html#setup-name, installing.html#requirements-files, + specifications.html#specifications, distributing.html#setup-cfg, index.html, glossary.html#term-extension-module, + distributing.html#setup-py, search.html#searchindexloader, specifications.html#provides-extra-multiple-use, + current.html, extensions.html#contents, deployment.html#application-bundles, extensions.html#alternatives-to-handcoded-accelerator-modules, + single_source_version.html, index.html#python-packaging-user-guide, glossary.html, + extensions.html#disadvantages, quickstart.html, contribute.html#scope] diff --git a/scripts/linkmonitor/linkmonitor.py b/scripts/linkmonitor/linkmonitor.py new file mode 100644 index 000000000..12d7ec1cb --- /dev/null +++ b/scripts/linkmonitor/linkmonitor.py @@ -0,0 +1,138 @@ +# Copyright 2017, PyPA +# The Python Packaging User Guide is licensed under a Creative Commons +# Attribution-ShareAlike license: +# http://creativecommons.org/licenses/by-sa/3.0. + +import argparse +from glob import glob +import io +import os +import sys + +from bs4 import BeautifulSoup +import yaml + +HERE = os.path.abspath(os.path.dirname(__file__)) +INVENTORY_FILENAME = os.path.join(HERE, 'inventory.yaml') +ROOT = os.path.abspath(os.path.join(HERE, '..', '..')) +HTML_DIR = os.path.join(ROOT, 'build', 'html') +IGNORED_FILES = [ + 'genindex.html' +] + + +def find_all_named_anchors(filename): + links = set() + + with io.open(filename, 'r') as html_file: + soup = BeautifulSoup(html_file, 'html.parser') + + for tag in soup.find_all(id=True): + anchor = tag['id'] + # Ignore non-named IDs. + if anchor.startswith('id'): + continue + # Ignore index anchors + if anchor.startswith('index-'): + continue + # Ignore searchbox anchors + if anchor == 'searchbox': + continue + + href = '{}#{}'.format(filename, anchor) + links.add(href) + + return links + + +def find_all_named_anchors_in_files(files): + links = set() + + for filename in files: + links.add(filename) + anchors = find_all_named_anchors(filename) + links.update(anchors) + + return links + + +def find_links(): + files = glob('**/*.html', recursive=True) + files = filter(lambda name: name not in IGNORED_FILES, files) + return find_all_named_anchors_in_files(files) + + +def load_inventory(): + if not os.path.exists(INVENTORY_FILENAME): + return set() + with io.open(INVENTORY_FILENAME, 'r') as inventory_file: + return set(yaml.load(inventory_file)) + + +def save_inventory(inventory): + with io.open(INVENTORY_FILENAME, 'w') as inventory_file: + yaml.dump(list(inventory), inventory_file) + + +def update_command(args): + """Updates the current inventory of links with any new links added. + + This should be run after adding new documentation to make a record of new + items added. + """ + os.chdir(HTML_DIR) + + inventory = load_inventory() + links = find_links() + + new_links = links.difference(inventory) + print('Found {} new links.'.format(len(new_links))) + + inventory.update(links) + save_inventory(inventory) + + return 0 + + +def check_command(args): + """Checks the current set of links against the inventory. + + This should be run on every documentation change to ensure that no deep + links have been broken. + """ + os.chdir(HTML_DIR) + + # TODO: Add another file to list currently defined redirects. + inventory = load_inventory() + links = find_links() + + missing_links = inventory.difference(links) + + if not missing_links: + print('All is well') + return 0 + + print('Missing the following deep links:') + for link in missing_links: + print(' * {}'.format(link)) + + return 1 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + update_parser = subparsers.add_parser( + 'update', help=update_command.__doc__) + update_parser.set_defaults(func=update_command) + check_parser = subparsers.add_parser( + 'check', help=check_command.__doc__) + check_parser.set_defaults(func=check_command) + + args = parser.parse_args() + + if not hasattr(args, 'func'): + parser.print_help() + sys.exit(1) + + sys.exit(args.func(args)) diff --git a/scripts/linkmonitor/requirements.txt b/scripts/linkmonitor/requirements.txt new file mode 100644 index 000000000..0aed6e7b2 --- /dev/null +++ b/scripts/linkmonitor/requirements.txt @@ -0,0 +1,2 @@ +beautifulsoup4==4.6.0 +PyYAML==3.12 From c2e38dbde3e89241dde662e1735efaa16cf57953 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 23 May 2017 20:15:37 -0700 Subject: [PATCH 2/2] Add redirect checking --- .travis.yml | 2 +- nox.py | 17 ++- scripts/linkmonitor/inventory.yaml | 191 +++++++++++++++-------------- scripts/linkmonitor/linkmonitor.py | 70 +++++++++-- scripts/linkmonitor/redirects.yaml | 7 ++ 5 files changed, 178 insertions(+), 109 deletions(-) create mode 100644 scripts/linkmonitor/redirects.yaml diff --git a/.travis.yml b/.travis.yml index 972bdb771..5b8c18230 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,4 +3,4 @@ sudo: false python: - 3.6 install: pip install --upgrade nox-automation virtualenv -script: nox +script: nox -s build checklinks diff --git a/nox.py b/nox.py index 9e2c95753..4da95ec35 100644 --- a/nox.py +++ b/nox.py @@ -14,14 +14,23 @@ def build(session): session.install('-r', 'requirements.txt') # Treat warnings as errors. session.env['SPHINXOPTS'] = '-W' - session.run('make', 'html') + session.run('make', 'clean', 'html') -@nox.session -def checklinks(session): +def linkmonitor(session, command): if not os.path.exists(os.path.join('build', 'html')): session.error('HTML output not available, run nox -s build first.') session.interpreter = 'python3.6' session.install('-r', 'scripts/linkmonitor/requirements.txt') session.run( - 'python', 'scripts/linkmonitor/linkmonitor.py', 'check') + 'python', 'scripts/linkmonitor/linkmonitor.py', command) + + +@nox.session +def checklinks(session): + linkmonitor(session, 'check') + + +@nox.session +def updatelinks(session): + linkmonitor(session, 'update') diff --git a/scripts/linkmonitor/inventory.yaml b/scripts/linkmonitor/inventory.yaml index a359ac9ea..13d38d44e 100644 --- a/scripts/linkmonitor/inventory.yaml +++ b/scripts/linkmonitor/inventory.yaml @@ -1,96 +1,97 @@ -[science.html, self_hosted_repository.html#hosting-your-own-simple-repository, pip_easy_install.html#pip-vs-easy-install, - glossary.html#term-binary-distribution, specifications.html#recording-installed-distributions, - glossary.html#term-requirement, namespace_packages.html#creating-a-namespace-package, - deployment.html#contents, contribute.html#voice-and-tone, platforms.html, wheel_egg.html, - distributing.html#pure-python-wheels, appveyor.html#setting-up, glossary.html#term-project, - distributing.html#packaging-and-distributing-projects, deployment.html#supporting-multiple-hardware-platforms, - glossary.html#term-release, key_projects.html#warehouse, distributing.html#package-data, - distributing.html#uploading-your-project-to-pypi, distributing.html#packages, key_projects.html#python-packaging-user-guide, - key_projects.html#virtualenv, glossary.html#term-per-project-index, install_requirements_linux.html, - extensions.html, key_projects.html#bandersnatch, glossary.html#term-working-set, - key_projects.html#project-summaries, installing.html#optionally-create-a-virtual-environment, - distributing.html#platform-wheels, key_projects.html#buildout, namespace_packages.html#native-namespace-packages, - installing.html#creating-virtual-environments, installing.html#creating-and-using-virtual-environments, - distributing.html#choosing-a-versioning-scheme, key_projects.html#devpi, glossary.html#term-version-specifier, - distributing.html#universal-wheels, distributing.html#date-based-versioning, specifications.html#package-distribution-metadata, - specifications.html#dependency-specifiers, patching.html#patching-forking, installing.html#installing-from-a-local-src-tree, - distributing.html#scheme-choices, key_projects.html#wheel, key_projects.html#distlib, - science.html#the-conda-cross-platform-package-manager, additional.html#additional-topics, - appveyor.html#testing-with-tox, current.html#packaging-tool-recommendations, distributing.html#wheels, - extensions.html#use-cases, wheel_egg.html#wheel-vs-egg, distributing.html#url, science.html#contents, - distributing.html#install-requires, search.html#fallback, distributing.html#version, - glossary.html#term-requirements-file, installing.html#upgrading-packages, glossary.html#term-python-packaging-authority-pypa, - distributing.html#keywords, distributing.html#manifest-in, key_projects.html#spack, - installing.html#installing-from-local-archives, installing.html#installing-to-the-user-site, - science.html#mac-os-x-installers-and-package-managers, multiple_python_versions.html#automated-testing-and-continuous-integration, - specifications.html#pypa-specifications, plugin_discovery.html#using-namespace-packages, - search.html#search-documentation, search.html#search-results, distributing.html#standards-compliance-for-interoperability, - multiple_python_versions.html#contents, installing.html#installing-packages, distributing.html#license, - extensions.html#alternatives-to-handcoded-wrapper-modules, extensions.html#alternatives-for-low-level-system-access, - requirements.html#contents, extensions.html#building-binary-extensions, specifications.html#package-index-interfaces, - distributing.html#local-version-identifiers, installing.html#source-distributions-vs-wheels, - distributing.html#name, extensions.html#an-overview-of-binary-extensions, key_projects.html#setuptools, - multi_version_install.html, install_requirements_linux.html#fedora, distributing.html#serial-versioning, - quickstart.html#quickstart, contribute.html#conventions-and-mechanics, deployment.html#configuration-management, - single_source_version.html#single-sourcing-the-project-version, installing.html#installing-prereleases, - appveyor.html#appveyor-yml, requirements.html#install-requires, glossary.html#term-module, - key_projects.html#twine, key_projects.html#easy-install, appveyor.html#contents, - plugin_discovery.html#using-naming-convention, science.html#scipy-distributions, - glossary.html#term-system-package, glossary.html#term-virtual-environment, specifications.html#binary-distribution-format, - self_hosted_repository.html, distributing.html#data-files, tutorial.html, multiple_python_versions.html#supporting-multiple-python-versions, - plugin_discovery.html#using-package-metadata, support.html#how-to-get-support, requirements.html#install-requires-vs-requirements-files, - distributing.html#working-in-development-mode, contribute.html#audience, distributing.html#console-scripts, - distributing.html#source-distributions, installing.html#installing-from-other-sources, - distributing.html#hybrid-schemes, appveyor.html#access-to-the-built-wheels, installing.html#install-pip-setuptools-and-wheel, - distributing.html#requirements-for-packaging-and-distributing, glossary.html#term-source-archive, - appveyor.html#external-dependencies, install_requirements_linux.html#centos-rhel, - key_projects.html#ensurepip, installing.html#installing-from-pypi, appveyor.html#additional-notes, - install_requirements_linux.html#debian-ubuntu, key_projects.html#pypa-projects, - patching.html, appveyor.html#supporting-windows-using-appveyor, namespace_packages.html#packaging-namespace-packages, - requirements.html, distributing.html#classifiers, key_projects.html#distutils, current.html#installation-tool-recommendations, - multiple_python_versions.html#what-s-in-which-python, additional.html, distributing.html#entry-points, - distributing.html#setup-args, key_projects.html#venv, specifications.html#core-metadata, - pip_easy_install.html, science.html#spack, science.html#linux-distribution-packages, - installing.html#installing-from-vcs, install_requirements_linux.html#opensuse, key_projects.html#packaging, +[additional.html, additional.html#additional-topics, appveyor.html, appveyor.html#access-to-the-built-wheels, + appveyor.html#adding-appveyor-support-to-your-project, appveyor.html#additional-notes, + appveyor.html#appveyor-yml, appveyor.html#automatically-uploading-wheels, appveyor.html#background, + appveyor.html#contents, appveyor.html#external-dependencies, appveyor.html#setting-up, + appveyor.html#support-script, appveyor.html#support-scripts, appveyor.html#supporting-windows-using-appveyor, + appveyor.html#testing-with-tox, contribute.html, contribute.html#audience, contribute.html#contribute-to-this-guide, + contribute.html#contributing-style-guide, contribute.html#conventions-and-mechanics, + contribute.html#purpose, contribute.html#scope, contribute.html#style-guide, contribute.html#voice-and-tone, + current.html, current.html#installation-tool-recommendations, current.html#packaging-tool-recommendations, + current.html#tool-recommendations, deployment.html, deployment.html#application-bundles, + deployment.html#application-deployment, deployment.html#configuration-management, + deployment.html#contents, deployment.html#os-packaging-installers, deployment.html#overview, + deployment.html#pynsist, deployment.html#supporting-multiple-hardware-platforms, + deployment.html#windows, distributing.html, distributing.html#author, distributing.html#choosing-a-versioning-scheme, + distributing.html#classifiers, distributing.html#configuring-your-project, distributing.html#console-scripts, + distributing.html#contents, distributing.html#create-an-account, distributing.html#data-files, + distributing.html#date-based-versioning, distributing.html#description, distributing.html#entry-points, + distributing.html#hybrid-schemes, distributing.html#initial-files, distributing.html#install-requires, + distributing.html#keywords, distributing.html#license, distributing.html#local-version-identifiers, + distributing.html#manifest-in, distributing.html#name, distributing.html#package-data, + distributing.html#packages, distributing.html#packaging-and-distributing-projects, + distributing.html#packaging-your-project, distributing.html#platform-wheels, distributing.html#pre-release-versioning, + distributing.html#pure-python-wheels, distributing.html#readme-rst, distributing.html#requirements-for-packaging-and-distributing, + distributing.html#scheme-choices, distributing.html#scripts, distributing.html#semantic-versioning-preferred, + distributing.html#serial-versioning, distributing.html#setup-args, distributing.html#setup-cfg, + distributing.html#setup-name, distributing.html#setup-py, distributing.html#source-distributions, + distributing.html#standards-compliance-for-interoperability, distributing.html#universal-wheels, + distributing.html#upload-your-distributions, distributing.html#uploading-your-project-to-pypi, + distributing.html#url, distributing.html#version, distributing.html#wheels, distributing.html#working-in-development-mode, + distributing.html#your-package, extensions.html, extensions.html#alternatives-for-low-level-system-access, + extensions.html#alternatives-to-handcoded-accelerator-modules, extensions.html#alternatives-to-handcoded-wrapper-modules, + extensions.html#an-overview-of-binary-extensions, extensions.html#binary-extensions, + extensions.html#building-binary-extensions, extensions.html#contents, extensions.html#disadvantages, + extensions.html#implementing-binary-extensions, extensions.html#publishing-binary-extensions, + extensions.html#setting-up-a-build-environment-on-windows, extensions.html#use-cases, + glossary.html, glossary.html#glossary, glossary.html#term-binary-distribution, glossary.html#term-built-distribution, + glossary.html#term-distribution-package, glossary.html#term-egg, glossary.html#term-extension-module, + glossary.html#term-import-package, glossary.html#term-known-good-set-kgs, glossary.html#term-module, + glossary.html#term-package-index, glossary.html#term-per-project-index, glossary.html#term-project, + glossary.html#term-pure-module, glossary.html#term-python-package-index-pypi, glossary.html#term-python-packaging-authority-pypa, + glossary.html#term-release, glossary.html#term-requirement, glossary.html#term-requirement-specifier, + glossary.html#term-requirements-file, glossary.html#term-setup-py, glossary.html#term-source-archive, + glossary.html#term-source-distribution-or-sdist, glossary.html#term-system-package, + glossary.html#term-version-specifier, glossary.html#term-virtual-environment, glossary.html#term-wheel, + glossary.html#term-working-set, index.html, index.html#python-packaging-user-guide, + install_requirements_linux.html, install_requirements_linux.html#arch-linux, install_requirements_linux.html#centos-rhel, + install_requirements_linux.html#debian-ubuntu, install_requirements_linux.html#fedora, install_requirements_linux.html#installing-pip-setuptools-wheel-with-linux-package-managers, - installing.html#requirements-for-installing-packages, glossary.html#term-requirement-specifier, - multi_version_install.html#multi-version-installs, support.html, namespace_packages.html, - contribute.html#style-guide, specifications.html#version-specifiers, specifications.html#platform-compatibility-tags, - distributing.html#readme-rst, namespace_packages.html#pkg-resources-style-namespace-packages, - distributing.html#scripts, contribute.html#contributing-style-guide, distributing.html#pre-release-versioning, - current.html#tool-recommendations, installing.html#use-pip-for-installing, platforms.html#platform-integtation, - glossary.html#term-python-package-index-pypi, contribute.html, glossary.html#term-source-distribution-or-sdist, - extensions.html#setting-up-a-build-environment-on-windows, glossary.html#term-known-good-set-kgs, - mirrors.html#contents, distributing.html#upload-your-distributions, installing.html#installing-requirements, - multiple_python_versions.html#tools-for-single-source-python-packages, deployment.html#windows, - mirrors.html#caching-with-pip, specifications.html#description-content-type, appveyor.html, - contribute.html#purpose, distributing.html#contents, requirements.html#requirements-files, - glossary.html#term-package-index, appveyor.html#background, mirrors.html#pypi-mirrors-and-caches, - key_projects.html#pip, mirrors.html#caching-with-devpi, key_projects.html#hashdist, - distributing.html#your-package, installing.html#installing-from-other-indexes, search.html, - plugin_discovery.html, mirrors.html#complete-mirror-with-bandersnatch, mirrors.html, - distributing.html#author, install_requirements_linux.html#arch-linux, distributing.html, - installing.html, specifications.html, glossary.html#term-setup-py, extensions.html#implementing-binary-extensions, - glossary.html#term-egg, plugin_discovery.html#plugin-creation-and-discovery, glossary.html#glossary, - deployment.html#application-deployment, deployment.html#os-packaging-installers, - glossary.html#term-built-distribution, installing.html#contents, deployment.html#pynsist, - glossary.html#term-wheel, multiple_python_versions.html, contribute.html#contribute-to-this-guide, - science.html#building-from-source, specifications.html#simple-repository-api, deployment.html#overview, - key_projects.html#bento, key_projects.html, appveyor.html#adding-appveyor-support-to-your-project, - distributing.html#packaging-your-project, appveyor.html#support-script, namespace_packages.html#pkgutil-style-namespace-packages, - extensions.html#binary-extensions, key_projects.html#conda, installing.html#installing-setuptools-extras, - distributing.html#configuring-your-project, key_projects.html#pex, key_projects.html#non-pypa-projects, - self_hosted_repository.html#manual-repository, distributing.html#create-an-account, - key_projects.html#projects, search.html#search-progress, science.html#windows-installers, - key_projects.html#standard-library-projects, distributing.html#initial-files, distributing.html#description, - glossary.html#term-distribution-package, appveyor.html#support-scripts, specifications.html#declaring-build-system-dependencies, - deployment.html, extensions.html#publishing-binary-extensions, glossary.html#term-pure-module, - glossary.html#term-import-package, appveyor.html#automatically-uploading-wheels, - single_source_version.html#single-sourcing-the-version, science.html#numpy-and-the-science-stack, - science.html#installing-scientific-packages, specifications.html#source-distribution-format, - distributing.html#semantic-versioning-preferred, distributing.html#setup-name, installing.html#requirements-files, - specifications.html#specifications, distributing.html#setup-cfg, index.html, glossary.html#term-extension-module, - distributing.html#setup-py, search.html#searchindexloader, specifications.html#provides-extra-multiple-use, - current.html, extensions.html#contents, deployment.html#application-bundles, extensions.html#alternatives-to-handcoded-accelerator-modules, - single_source_version.html, index.html#python-packaging-user-guide, glossary.html, - extensions.html#disadvantages, quickstart.html, contribute.html#scope] + install_requirements_linux.html#opensuse, installing.html, installing.html#contents, + installing.html#creating-and-using-virtual-environments, installing.html#creating-virtual-environments, + installing.html#install-pip-setuptools-and-wheel, installing.html#installing-from-a-local-src-tree, + installing.html#installing-from-local-archives, installing.html#installing-from-other-indexes, + installing.html#installing-from-other-sources, installing.html#installing-from-pypi, + installing.html#installing-from-vcs, installing.html#installing-packages, installing.html#installing-prereleases, + installing.html#installing-requirements, installing.html#installing-setuptools-extras, + installing.html#installing-to-the-user-site, installing.html#optionally-create-a-virtual-environment, + installing.html#requirements-files, installing.html#requirements-for-installing-packages, + installing.html#source-distributions-vs-wheels, installing.html#upgrading-packages, + installing.html#use-pip-for-installing, key_projects.html, key_projects.html#bandersnatch, + key_projects.html#bento, key_projects.html#buildout, key_projects.html#conda, key_projects.html#devpi, + key_projects.html#distlib, key_projects.html#distutils, key_projects.html#easy-install, + key_projects.html#ensurepip, key_projects.html#hashdist, key_projects.html#non-pypa-projects, + key_projects.html#packaging, key_projects.html#pex, key_projects.html#pip, key_projects.html#project-summaries, + key_projects.html#projects, key_projects.html#pypa-projects, key_projects.html#python-packaging-user-guide, + key_projects.html#setuptools, key_projects.html#spack, key_projects.html#standard-library-projects, + key_projects.html#twine, key_projects.html#venv, key_projects.html#virtualenv, key_projects.html#warehouse, + key_projects.html#wheel, mirrors.html, mirrors.html#caching-with-devpi, mirrors.html#caching-with-pip, + mirrors.html#complete-mirror-with-bandersnatch, mirrors.html#contents, mirrors.html#pypi-mirrors-and-caches, + multi_version_install.html, multi_version_install.html#multi-version-installs, multiple_python_versions.html, + multiple_python_versions.html#automated-testing-and-continuous-integration, multiple_python_versions.html#contents, + multiple_python_versions.html#supporting-multiple-python-versions, multiple_python_versions.html#tools-for-single-source-python-packages, + multiple_python_versions.html#what-s-in-which-python, namespace_packages.html, namespace_packages.html#creating-a-namespace-package, + namespace_packages.html#native-namespace-packages, namespace_packages.html#packaging-namespace-packages, + namespace_packages.html#pkg-resources-style-namespace-packages, namespace_packages.html#pkgutil-style-namespace-packages, + patching.html, patching.html#patching-forking, pip_easy_install.html, pip_easy_install.html#pip-vs-easy-install, + platforms.html, platforms.html#platform-integtation, plugin_discovery.html, plugin_discovery.html#plugin-creation-and-discovery, + plugin_discovery.html#using-namespace-packages, plugin_discovery.html#using-naming-convention, + plugin_discovery.html#using-package-metadata, quickstart.html, quickstart.html#quickstart, + requirements.html, requirements.html#contents, requirements.html#install-requires, + requirements.html#install-requires-vs-requirements-files, requirements.html#requirements-files, + science.html, science.html#building-from-source, science.html#contents, science.html#installing-scientific-packages, + science.html#linux-distribution-packages, science.html#mac-os-x-installers-and-package-managers, + science.html#numpy-and-the-science-stack, science.html#scipy-distributions, science.html#spack, + science.html#the-conda-cross-platform-package-manager, science.html#windows-installers, + search.html, search.html#fallback, search.html#search-documentation, search.html#search-progress, + search.html#search-results, search.html#searchindexloader, self_hosted_repository.html, + self_hosted_repository.html#hosting-your-own-simple-repository, self_hosted_repository.html#manual-repository, + single_source_version.html, single_source_version.html#single-sourcing-the-project-version, + single_source_version.html#single-sourcing-the-version, specifications.html, specifications.html#binary-distribution-format, + specifications.html#core-metadata, specifications.html#declaring-build-system-dependencies, + specifications.html#dependency-specifiers, specifications.html#description-content-type, + specifications.html#package-distribution-metadata, specifications.html#package-index-interfaces, + specifications.html#platform-compatibility-tags, specifications.html#provides-extra-multiple-use, + specifications.html#pypa-specifications, specifications.html#recording-installed-distributions, + specifications.html#simple-repository-api, specifications.html#source-distribution-format, + specifications.html#specifications, specifications.html#version-specifiers, support.html, + support.html#how-to-get-support, tutorial.html, wheel_egg.html, wheel_egg.html#wheel-vs-egg] diff --git a/scripts/linkmonitor/linkmonitor.py b/scripts/linkmonitor/linkmonitor.py index 12d7ec1cb..fa3cfe839 100644 --- a/scripts/linkmonitor/linkmonitor.py +++ b/scripts/linkmonitor/linkmonitor.py @@ -14,6 +14,7 @@ HERE = os.path.abspath(os.path.dirname(__file__)) INVENTORY_FILENAME = os.path.join(HERE, 'inventory.yaml') +REDIRECTS_FILENAME = os.path.join(HERE, 'redirects.yaml') ROOT = os.path.abspath(os.path.join(HERE, '..', '..')) HTML_DIR = os.path.join(ROOT, 'build', 'html') IGNORED_FILES = [ @@ -71,7 +72,40 @@ def load_inventory(): def save_inventory(inventory): with io.open(INVENTORY_FILENAME, 'w') as inventory_file: - yaml.dump(list(inventory), inventory_file) + yaml.dump(sorted(list(inventory)), inventory_file) + + +def load_redirects(): + with io.open(REDIRECTS_FILENAME, 'r') as redirects_file: + return yaml.load(redirects_file) + + +def expand_redirects(redirects, inventory): + valid_redirects = set() + missing_redirects = set() + + for redirect in redirects: + from_ = redirect['from'] + source_links = set() + + # Get all links that start with the page. This gathers all deep links. + # For example, the redirect may be old.html -> new.html. old.html may + # have had #1, #2, #3. We need to get all of those deep links. + for link in inventory: + if link.startswith(from_): + source_links.add(link) + + # Make sure all of the source links have a counterpart in the + # destination page. For the example above, new.html needs to have #1 + # #2 and #3 as well. + for source_link in source_links: + dest_link = source_link.replace(from_, redirect['to']) + if dest_link in inventory: + valid_redirects.add(source_link) + else: + missing_redirects.add((source_link, dest_link)) + + return valid_redirects, missing_redirects def update_command(args): @@ -98,25 +132,43 @@ def check_command(args): """Checks the current set of links against the inventory. This should be run on every documentation change to ensure that no deep - links have been broken. + links have been broken and that new links are tracked in the inventory. """ os.chdir(HTML_DIR) # TODO: Add another file to list currently defined redirects. inventory = load_inventory() + redirects = load_redirects() links = find_links() + valid_redirects, missing_redirects = expand_redirects(redirects, inventory) + if missing_redirects: + print( + 'The following redirects are missing deep link anchors in the ' + 'destination:') + for source, dest in missing_redirects: + print(' * {} -> {}'.format(source, dest)) + missing_links = inventory.difference(links) + missing_links -= valid_redirects - if not missing_links: - print('All is well') - return 0 + if missing_links: + print('Missing the following deep links:') + for link in missing_links: + print(' * {}'.format(link)) + return 1 - print('Missing the following deep links:') - for link in missing_links: - print(' * {}'.format(link)) + new_links = links.difference(inventory) + + if new_links: + print('The following new deep links were added:') + for link in new_links: + print(' * {}'.format(link)) + print('Run nox -s updatelinks to update them in git.') + return 2 - return 1 + print('All is well') + return 0 if __name__ == '__main__': diff --git a/scripts/linkmonitor/redirects.yaml b/scripts/linkmonitor/redirects.yaml new file mode 100644 index 000000000..87eaffea6 --- /dev/null +++ b/scripts/linkmonitor/redirects.yaml @@ -0,0 +1,7 @@ +# Unfortunately, readthedocs doesn't allow us to specify redirects in yaml. +# They have to be individually added in the readthedocs UI. This file should +# match the configuration of page redirects in the UI. +# See also: https://github.com/rtfd/readthedocs.org/issues/2904 + +- from: old.html + to: new.html