Ryan CurtinandDirk Eddelbuettel
9fd815144d
* A first attempt at overhauling the documentation job with a cache. * Actually remove files we don't need anymore. * Fix output from local link checks to give it all at once. * Add sparse matrices to the set of detected matrix types. * Fix comparison condition. * Make a pass to try and re-validate cached links that have not yet expired. * Compile and link separately so that ccache can take effect. * Be a little bit smarter about caching. * Ensure libicu$SO is installed so 'stringi' can be used Also collapse to remotes calls into one * Fix style issues that cpplint 2 found (#3884) * Try to workaround the libicu issue for now. * Mention new dependency. --------- Co-authored-by: Dirk Eddelbuettel <edd@debian.org>
758 lines
26 KiB
Bash
Executable File
758 lines
26 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Convert all the Markdown files in doc/ to HTML.
|
|
#
|
|
# This requires `kramdown` to be available on the path. `tidy` and
|
|
# `linkchecker` (the Python package) and `checklink` (from w3c-linkchecker on
|
|
# Debian) are used to test the output and must also be available and on the
|
|
# path. `sqlite3` must also be available.
|
|
#
|
|
# Run this from the root directory of the repository.
|
|
#
|
|
# The output directory can be specified as the first option.
|
|
#
|
|
# If the environment variable DISABLE_HTML_CHECKS is specified, then checks are
|
|
# skipped.
|
|
#
|
|
# If the environment variable LINK_CACHE_FILE is specified, then that file is
|
|
# used as a cache of already-valid links that will not be checked.
|
|
|
|
if [ "$#" -gt 1 ]; then
|
|
echo "Usage: $0 [output_dir/]";
|
|
exit 1;
|
|
elif [ "$#" -eq 1 ]; then
|
|
output_dir=$1;
|
|
else
|
|
output_dir=doc/html;
|
|
fi
|
|
|
|
# If the header and footer already exist, they will not be overwritten.
|
|
template_html_header="${output_dir}/template.html.header";
|
|
template_html_footer="${output_dir}/template.html.footer";
|
|
template_html_sidebar="${output_dir}/template.html.sidebar";
|
|
|
|
if ! command -v kramdown &>/dev/null
|
|
then
|
|
echo "kramdown not installed! Cannot build documentation.";
|
|
exit 1;
|
|
fi
|
|
|
|
# If DISABLE_HTML_CHECKS is set, then we won't use tidy or checklink.
|
|
if [ -z ${DISABLE_HTML_CHECKS+x} ];
|
|
then
|
|
if ! command -v tidy &>/dev/null
|
|
then
|
|
echo "tidy not installed! Cannot check documentation.";
|
|
exit 1;
|
|
fi
|
|
|
|
if ! command -v checklink &>/dev/null
|
|
then
|
|
echo "checklink not installed! Cannot check documentation.";
|
|
exit 1;
|
|
fi
|
|
|
|
if ! command -v linkchecker &>/dev/null
|
|
then
|
|
echo "linkchecker not installed! Cannot check documentation.";
|
|
exit 1;
|
|
fi
|
|
|
|
if ! command -v sqlite3 &> /dev/null
|
|
then
|
|
echo "sqlite3 not installed! Cannot check documentation.";
|
|
exit 1;
|
|
fi
|
|
fi
|
|
|
|
if [ ! -d doc/ ];
|
|
then
|
|
echo "Run this script from the root of the mlpack repository.";
|
|
exit 1;
|
|
fi
|
|
|
|
# Define utility function to run kramdown and turn an .md file to an .html file.
|
|
run_kramdown()
|
|
{
|
|
input_file=$1;
|
|
# This converts, e.g., ./doc/user/index.md -> doc/html/user/index.html.
|
|
tmp=${input_file#./doc/}; # Strip leading ./doc/.
|
|
output_file="$output_dir/${tmp%.md}.html";
|
|
|
|
# Determine what the link root is. If we're in the root directory, it's
|
|
# nothing, otherwise it's one of more '../'s.
|
|
dir_name=$(dirname $tmp);
|
|
link_root="";
|
|
if [[ "$dir_name" != "." ]];
|
|
then
|
|
levels_below_root=`echo $dir_name | awk -F'/' '{ print NF }'`;
|
|
link_root=$(printf '../%.0s' `seq 1 $levels_below_root`);
|
|
fi
|
|
|
|
# Make the enclosing directory if needed.
|
|
out_dir=`dirname "$output_file"`;
|
|
mkdir -p "$out_dir";
|
|
|
|
# Kramdown doesn't detect languages correctly with the "```" fence; instead it
|
|
# needs the "~~~" fence.
|
|
sed 's/^```/~~~/' $input_file > $input_file.tmp;
|
|
|
|
# Our documentation is full of relative links, like
|
|
# [name](other_file.md#anchor). We need these to turn into links to the
|
|
# rendered HTML file, like [name](other_file.html#anchor). We'll do this with
|
|
# regular expressions...
|
|
#
|
|
# - Note that this assumes there are no spaces in any filenames.
|
|
# - We also only catch the second part of the link '](' because the name of
|
|
# the link could be spread on multiple lines.
|
|
#
|
|
# We start by trying to catch the special cases README.md and HISTORY.md,
|
|
# which our documentation puts in a slightly different place. In addition,
|
|
# because those files are being moved to the root of the documentation, we
|
|
# must adjust links differently.
|
|
if [[ $input_file != "README.md" ]] && [[ $input_file != "HISTORY.md" ]];
|
|
then
|
|
sed -i "s|\]([./]*README.md)|](${link_root}README.html)|g" $input_file.tmp;
|
|
sed -i "s|\]([./]*README.md#[0-9]-\([^ ]*\))|](${link_root}README.html#\1)|g" $input_file.tmp;
|
|
sed -i 's/\](\([^ ]*\).md)/](\1.html)/g' $input_file.tmp;
|
|
sed -i 's/\](\([^ ]*\).md#\([^ ]*\))/](\1.html#\2)/g' $input_file.tmp;
|
|
else
|
|
sed -i 's/\](doc\/\([^ ]*\).md)/](\1.html)/g' $input_file.tmp;
|
|
sed -i 's/\](doc\/\([^ ]*\).md#\([^ ]*\))/](\1.html#\2)/g' $input_file.tmp;
|
|
|
|
# The README specifically has a link to GOVERNANCE.md, but we want to
|
|
# preserve that. We're not building that file into Markdown.
|
|
sed -i 's|(./GOVERNANCE.md)|(https://github.com/mlpack/mlpack/blob/master/GOVERNANCE.md)|' $input_file.tmp;
|
|
|
|
# Ugh! Github naming of anchors is different than kramdown, and so we have
|
|
# to adjust all the table-of-contents anchor links in the README (and in
|
|
# that file only).
|
|
sed -i 's/\](#[0-9][0-9]-\([^ ]*\))/](#\1)/g' $input_file.tmp;
|
|
sed -i 's/\](#[0-9]-\([^ ]*\))/](#\1)/g' $input_file.tmp;
|
|
sed -i 's/\](#[0-9][0-9]\([^ ]*\))/](#\1)/g' $input_file.tmp;
|
|
sed -i 's/\](#[0-9]\([^ ]*\))/](#\1)/g' $input_file.tmp;
|
|
|
|
# For HISTORY.md, we want to turn all references to Github issues into
|
|
# actual links, and all references to Github usernames into links to their
|
|
# profile.
|
|
if [[ $input_file == "HISTORY.md" ]];
|
|
then
|
|
sed -i 's/#\([0-9][0-9]*\)/[#\1](https:\/\/github.com\/mlpack\/mlpack\/issues\/\1)/g' $input_file.tmp;
|
|
sed -i 's/\([^`]\)@\([a-zA-Z0-9_-][a-zA-Z0-9_-]*\)/\1[@\2](https:\/\/github.com\/\2)/g' $input_file.tmp;
|
|
fi
|
|
fi
|
|
|
|
# Replace any links to source files with a link to the current version of the
|
|
# source file on Github.
|
|
sed -i 's/\](\/src\/\([^ ]*\)\.hpp)/](https:\/\/github.com\/mlpack\/mlpack\/blob\/master\/src\/\1.hpp)/' $input_file.tmp;
|
|
|
|
# If this is binding documentation or quickstart documentation, don't set the
|
|
# default language to C++.
|
|
set_lang=1;
|
|
if [[ `dirname $input_file` == "./doc/user/bindings" ]];
|
|
then
|
|
set_lang=0;
|
|
elif [[ `dirname $input_file` == "./doc/quickstart" ]];
|
|
then
|
|
if [[ `basename $input_file .md` != "cpp" ]];
|
|
then
|
|
set_lang=0;
|
|
fi
|
|
elif [[ $input_file == "HISTORY.md" ]];
|
|
then
|
|
set_lang=0;
|
|
fi
|
|
|
|
if [[ "$set_lang" == "0" ]];
|
|
then
|
|
kramdown \
|
|
-x parser-gfm \
|
|
--syntax-highlighter rouge \
|
|
--auto_ids \
|
|
$input_file.tmp > "$output_file.tmp" || exit 1;
|
|
else
|
|
kramdown \
|
|
-x parser-gfm \
|
|
--syntax-highlighter rouge \
|
|
--syntax-highlighter-opts '{ default_lang: c++ }' \
|
|
--auto_ids \
|
|
$input_file.tmp > "$output_file.tmp" || exit 1;
|
|
fi
|
|
cat "$template_html_header" | sed "s|LINKROOT|$link_root|" > "$output_file";
|
|
|
|
# Create the sidebar. Extract anchors from the page, unless we are looking at
|
|
# index.md, since the permanent part of the sidebar links all over index.md
|
|
# anyway. If we are looking at binding documentation, use a slightly
|
|
# different sidebar.
|
|
if { [[ $dir_name != "user/bindings" ]] && \
|
|
[[ $dir_name != "quickstart" ]] } ||
|
|
[[ $input_file == "./doc/quickstart/cpp.md" ]];
|
|
then
|
|
cat "$template_html_sidebar" | sed "s|LINKROOT|$link_root|" \
|
|
>> "$output_file";
|
|
create_page_sidebar_section "$output_file.tmp" "$output_file" "$dir_name";
|
|
else
|
|
echo "Using custom sidebar...";
|
|
cat "$template_html_sidebar" | sed "s|LINKROOT|$link_root|" |\
|
|
sed 's|<details> <!-- default closed for non-binding pages -->|<details open="true">|' |\
|
|
sed 's|<details open="true"> <!-- default open for non-binding pages -->|<details>|' \
|
|
>> "$output_file";
|
|
# Some pages may have a custom sidebar HTML file. (Specifically,
|
|
# generated language bindings.)
|
|
if [[ $dir_name == "user/bindings" ]];
|
|
then
|
|
cat "${input_file/%.md/.sidebar.html}" | sed "s|LINKROOT|$link_root|" \
|
|
>> "$output_file";
|
|
else
|
|
sidebar_file=`basename $input_file .md`.sidebar.html;
|
|
cat "./doc/user/bindings/$sidebar_file" | sed "s|LINKROOT|$link_root|" \
|
|
>> "$output_file";
|
|
fi
|
|
fi
|
|
|
|
# Add clickable anchors to h2 and h3 headers.
|
|
echo "<div id=\"content\">" >> "$output_file";
|
|
sed -E 's/<h([23]) id="([^"]*)">/<h\1 id="\2"><a href="#\2" class="pl">🔗<\/a> /' "$output_file.tmp" >> "$output_file";
|
|
|
|
# Simple postprocessing to make tidy a little happier.
|
|
# (Muting the warning won't change the error code!)
|
|
sed -i 's/<table>/<table summary="">/' "$output_file";
|
|
|
|
cat "$template_html_footer" >> "$output_file";
|
|
rm -f $input_file.tmp "$output_file.tmp";
|
|
}
|
|
|
|
# Create the template header file.
|
|
create_template_header()
|
|
{
|
|
output_file="$1";
|
|
|
|
# Note that LINKROOT will be substituted into place by run_kramdown.
|
|
cat > "$output_file" << EOF
|
|
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
|
<html>
|
|
<head>
|
|
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<link type="text/css" rel="stylesheet" href="LINKROOTgfm-mod.css">
|
|
<link type="text/css" rel="stylesheet" href="LINKROOTrouge-github-mod.css">
|
|
<title>mlpack documentation</title>
|
|
</head>
|
|
<body>
|
|
EOF
|
|
}
|
|
|
|
# Create the template footer.
|
|
create_template_footer()
|
|
{
|
|
output_file="$1";
|
|
|
|
cat > "$output_file" << EOF
|
|
</div>
|
|
</body>
|
|
</html>
|
|
EOF
|
|
}
|
|
|
|
# Extract anchors to build a sidebar.
|
|
# This should take the input HTML (before anchor elements are added), and it
|
|
# appends a sidebar list to the output HTML.
|
|
create_page_sidebar_section()
|
|
{
|
|
sb_input_file="$1";
|
|
sb_output_file="$2";
|
|
sb_dir_name="$3"; # The directory containing the documentation.
|
|
sb_input_file_base=`basename "$sb_input_file" .html.tmp`;
|
|
|
|
# Extract h2/h3 anchors into a list. For individual method documentation, we
|
|
# only extract h3 anchors because those use h2s as their headings. And, for
|
|
# core.md, we want to extract both h2 and h3 anchors.
|
|
if [[ "$sb_dir_name" == "user/methods" ]];
|
|
then
|
|
# The page title on individual methods is encoded as an h2.
|
|
page_title=`grep '<h2 id=' "$sb_input_file" |\
|
|
head -1 |\
|
|
sed 's/^<h2 id="[^"]*">\(.*\)<\/h2>/\1/'`;
|
|
|
|
grep '<h3 id=' "$sb_input_file" | sed 's/<h3 id="\([^"]*\)">\(.*\)<\/h3>/<li><a href="#\1">\2<\/a><\/li>/' > "$sb_output_file.side.tmp";
|
|
elif [[ "$sb_input_file_base" == "core" || "$sb_dir_name" == "user/core" || "$sb_dir_name" == "user/core/trees" ]];
|
|
then
|
|
# The page title on the core class documentation page is encoded as an h1.
|
|
page_title=`grep '<h1 id=' "$sb_input_file" |\
|
|
head -1 |\
|
|
sed 's/^<h1 id="[^"]*">\(.*\)<\/h1>/\1/'`;
|
|
|
|
# We want to collect h2s and h3s as individual documentation; each h2 should
|
|
# have a summary/details block. This is a little tedious to create... we'll
|
|
# do this by creating a temporary tab-separated file with lines like
|
|
#
|
|
# h2 anchor_name Anchor Title
|
|
# h3 anchor_name Anchor Title
|
|
# ...
|
|
#
|
|
# and then we'll construct the actual sidebar using that list.
|
|
grep '<h[23] id=' "$sb_input_file" |\
|
|
sed 's/^<\(h[23]\) id="\([^"]*\)">\(.*\)<\/h[23]>/\1\t\2\t\3/' \
|
|
> "$sb_output_file.side.list.tmp";
|
|
in_block=0;
|
|
while read line; do
|
|
# First, extract the pieces of each line.
|
|
line_type=`echo "$line" | awk -F'\t' '{ print $1 }'`;
|
|
anchor_name=`echo "$line" | awk -F'\t' '{ print $2 }'`;
|
|
anchor_title=`echo "$line" | awk -F'\t' '{ print $3 }'`;
|
|
|
|
# For an h2, we have to print a summary block, if the h2 has any children.
|
|
# (Below is a hacky way to detect that.)
|
|
h3_lines=`grep -A 1 "$line" "$sb_output_file.side.list.tmp" |\
|
|
tail -1 |\
|
|
grep 'h3' |\
|
|
wc -l`;
|
|
|
|
# Note that this assumes that *all* h2s have h3 children. If that's not
|
|
# true, some extra processing will be needed.
|
|
if [ "$line_type" = "h2" ]
|
|
then
|
|
# Close a block if necessary.
|
|
if [ "$in_block" = "1" ];
|
|
then
|
|
# We have to close the previous block.
|
|
echo "</ul></details></li>" >> "$sb_output_file.side.tmp";
|
|
in_block=0;
|
|
fi
|
|
|
|
# Create the new details block, if the h2 has children.
|
|
if [ "$h3_lines" -gt 0 ];
|
|
then
|
|
echo "<li><details><summary>" >> "$sb_output_file.side.tmp";
|
|
echo "<a href=\"#$anchor_name\">" >> "$sb_output_file.side.tmp";
|
|
echo "$anchor_title" >> "$sb_output_file.side.tmp";
|
|
echo "</a>" >> "$sb_output_file.side.tmp";
|
|
echo "</summary>" >> "$sb_output_file.side.tmp";
|
|
echo "<ul>" >> "$sb_output_file.side.tmp";
|
|
in_block=1;
|
|
else
|
|
echo " <li><a href=\"#$anchor_name\">" >> "$sb_output_file.side.tmp";
|
|
echo " $anchor_title" >> "$sb_output_file.side.tmp";
|
|
echo " </a></li>" >> "$sb_output_file.side.tmp";
|
|
fi
|
|
else
|
|
echo " <li><a href=\"#$anchor_name\">" >> "$sb_output_file.side.tmp";
|
|
echo " $anchor_title" >> "$sb_output_file.side.tmp";
|
|
echo " </a></li>" >> "$sb_output_file.side.tmp";
|
|
fi
|
|
done < "$sb_output_file.side.list.tmp";
|
|
|
|
# Close the last h2 block, if we need to.
|
|
if [ "$in_block" = "1" ];
|
|
then
|
|
echo "</ul></details></li>" >> "$sb_output_file.side.tmp";
|
|
fi
|
|
|
|
rm -f "$sb_output_file.side.list.tmp";
|
|
else
|
|
# On other pages, the page title is encoded as an h1.
|
|
page_title=`grep '<h1 id=' "$sb_input_file" |\
|
|
head -1 |\
|
|
sed 's/^<h1 id="[^"]*">\(.*\)<\/h1>/\1/'`;
|
|
|
|
grep '<h2 id=' "$sb_input_file" |\
|
|
sed 's/<h2 id="\([^"]*\)">\(.*\)<\/h2>/<li><a href="#\1">\2<\/a><\/li>/' \
|
|
> "$sb_output_file.side.tmp";
|
|
fi
|
|
lines=`cat "$sb_output_file.side.tmp" | wc -l`;
|
|
|
|
echo "<ul>" >> "$sb_output_file";
|
|
|
|
# Make the top of the sidebar.
|
|
if [ -n "$page_title" ];
|
|
then
|
|
echo "<li class=\"page_title\"><b>$page_title</b> <a href=\"#\">[top]</a>" >> "$sb_output_file";
|
|
else
|
|
echo "<li><a href=\"#\">[top of page]</a>" >> "$sb_output_file";
|
|
fi
|
|
|
|
if [[ "$lines" -gt 0 ]];
|
|
then
|
|
echo "<ul>" >> "$sb_output_file";
|
|
cat "$sb_output_file.side.tmp" >> "$sb_output_file";
|
|
echo "</ul>" >> "$sb_output_file";
|
|
fi
|
|
echo "</li>" >> "$sb_output_file";
|
|
echo "</ul>" >> "$sb_output_file";
|
|
echo "</div>" >> "$sb_output_file";
|
|
|
|
rm -f "$sb_output_file.side.tmp";
|
|
}
|
|
|
|
# Save any existing template.
|
|
if [ -f "$template_html_header" ];
|
|
then
|
|
mv "$template_html_header" template.html.header.tmp;
|
|
fi
|
|
|
|
if [ -f "$template_html_footer" ];
|
|
then
|
|
mv "$template_html_footer" template.html.footer.tmp;
|
|
fi
|
|
|
|
rm -rf "$output_dir";
|
|
mkdir -p "$output_dir";
|
|
cp -v doc/css/* "$output_dir";
|
|
mkdir -p "$output_dir/img/";
|
|
cp -v doc/img/* "$output_dir/img/";
|
|
mkdir -p "$output_dir/tutorials/res/";
|
|
cp -v doc/tutorials/res/* "$output_dir/tutorials/res/";
|
|
|
|
# Create the template files we will use, if they don't already exist.
|
|
if [ -f template.html.header.tmp ];
|
|
then
|
|
mv template.html.header.tmp "$template_html_header";
|
|
else
|
|
create_template_header "$template_html_header";
|
|
del_header=1;
|
|
fi
|
|
|
|
if [ -f template.html.footer.tmp ];
|
|
then
|
|
mv template.html.footer.tmp "$template_html_footer";
|
|
else
|
|
create_template_footer "$template_html_footer";
|
|
del_footer=1;
|
|
fi
|
|
|
|
cp doc/sidebar.html "$template_html_sidebar";
|
|
|
|
# Process all the .md files.
|
|
for f in README.md HISTORY.md `find ./doc/ -iname '*.md'`;
|
|
do
|
|
# Skip the JOSS paper...
|
|
if [[ $f == *"joss_paper"* ]]; then
|
|
continue;
|
|
fi
|
|
|
|
echo "Processing $f...";
|
|
run_kramdown $f;
|
|
|
|
# This converts, e.g., ./doc/user/index.md -> doc/html/user/index.html.
|
|
tmp=${f#./doc/}; # Strip leading ./doc/.
|
|
of="$output_dir/${tmp%.md}.html";
|
|
|
|
# Check HTML, if we did not disable that.
|
|
if [ -z ${DISABLE_HTML_CHECKS+x} ];
|
|
then
|
|
tidy -qe "$of" || exit 1;
|
|
fi
|
|
done
|
|
|
|
# Now take a second pass to check all local links.
|
|
if [ -z ${DISABLE_HTML_CHECKS+x} ];
|
|
then
|
|
find "$output_dir" -iname '*.html' -print0 | while read -d $'\0' f
|
|
do
|
|
echo "Checking local links and anchors in $f...";
|
|
|
|
# To run checklink we have to strip out some perl stderr warnings...
|
|
checklink -qs \
|
|
--follow-file-links \
|
|
--suppress-broken 405 \
|
|
--suppress-broken 503 \
|
|
--suppress-broken 301 \
|
|
--suppress-broken 400 \
|
|
-X "^http.*$" "$f" 2>&1 |
|
|
grep -v 'Use of uninitialized value' > checklink_out;
|
|
if [ -s checklink_out ];
|
|
then
|
|
# Store up all failures to print them at once.
|
|
cat checklink_out >> overall_checklink_out;
|
|
fi
|
|
rm -f checklink_out;
|
|
done
|
|
|
|
# Check to see if there were any failures, all at once.
|
|
if [ -f overall_checklink_out ];
|
|
then
|
|
cat overall_checklink_out;
|
|
rm -f overall_checklink_out;
|
|
exit 1;
|
|
fi
|
|
|
|
# Check to see if there were any failures, all at once.
|
|
fi
|
|
|
|
# Utility script to create linkchecker result SQL table, with a bit of extra
|
|
# information.
|
|
cat > create.sql << EOF
|
|
create table linksdb (
|
|
urlname varchar(256) not null,
|
|
parentname varchar(256),
|
|
baseref varchar(256),
|
|
valid int,
|
|
result varchar(256),
|
|
warning varchar(512),
|
|
info varchar(512),
|
|
url varchar(256),
|
|
line int,
|
|
col int,
|
|
name varchar(256),
|
|
checktime int,
|
|
dltime int,
|
|
size int,
|
|
cached int,
|
|
level int not null,
|
|
modified int,
|
|
resulttime timestamp,
|
|
validdays int
|
|
);
|
|
EOF
|
|
|
|
# Finally, take a third pass to check external links.
|
|
if [ -z ${DISABLE_HTML_CHECKS+x} ];
|
|
then
|
|
# Create a basic config file for linkchecker. We will append domains to ignore
|
|
# to this as we go.
|
|
echo "[checking]" > "$output_dir/linkcheckerrc.in";
|
|
echo "maxrequestspersecond=2" >> "$output_dir/linkcheckerrc.in";
|
|
echo "" >> "$output_dir/linkcheckerrc.in";
|
|
echo "[filtering]" >> "$output_dir/linkcheckerrc.in";
|
|
echo "ignore=" >> "$output_dir/linkcheckerrc.in";
|
|
echo " ^(?!http).*$" >> "$output_dir/linkcheckerrc.in";
|
|
# Github issues/pull requests redirect to each other and we link to so many of
|
|
# them it's not worth checking them.
|
|
echo " ^https://github.com/mlpack/mlpack/issues/[0-9]*$" >> "$output_dir/linkcheckerrc.in";
|
|
echo " ^https://github.com/mlpack/mlpack/issues[?]q.*$" >> "$output_dir/linkcheckerrc.in";
|
|
echo " ^https://github.com/mlpack/mlpack/pulls[?]q.*$" >> "$output_dir/linkcheckerrc.in";
|
|
|
|
# Initialize our cache or take the current version of it.
|
|
if [ ! -z ${LINK_CACHE_FILE+x} ];
|
|
then
|
|
if [ -f ${LINK_CACHE_FILE} ];
|
|
then
|
|
cp "$LINK_CACHE_FILE" "$output_dir/all_links.db";
|
|
else
|
|
rm -f "$output_dir/all_links.db";
|
|
cat create.sql | sqlite3 "$output_dir/all_links.db";
|
|
fi
|
|
else
|
|
rm -f "$output_dir/all_links.db";
|
|
cat create.sql | sqlite3 "$output_dir/all_links.db";
|
|
fi
|
|
|
|
find "$output_dir" -iname '*.html' -print0 | while read -d $'\0' f
|
|
do
|
|
echo "Checking external links in $f...";
|
|
|
|
# Generate our config file for this file by appending all valid files that
|
|
# we have already seen. Note that we have to append $ to all the ignore
|
|
# patterns so that we don't accidentally match anchors that haven't been
|
|
# checked yet.
|
|
cp "$output_dir/linkcheckerrc.in" "$output_dir/linkcheckerrc";
|
|
echo "SELECT DISTINCT urlname FROM linksdb
|
|
WHERE valid = 1 AND
|
|
urlname LIKE 'http%' AND
|
|
julianday(datetime()) - julianday(resulttime) < validdays AND
|
|
(result LIKE '200%' OR
|
|
result = 'filtered' OR
|
|
result = 'syntax OK');" | sqlite3 "$output_dir/all_links.db" |\
|
|
sed 's/^/ /' |\
|
|
sed 's/?/\\?/g' |\
|
|
sed 's/$/$/' >> "$output_dir/linkcheckerrc";
|
|
|
|
# Run linkchecker, and make things a little bit prettier if there are
|
|
# failures.
|
|
rm -f links.sql;
|
|
linkchecker --check-extern \
|
|
--recursion-level=1 \
|
|
--threads=4 \
|
|
--verbose \
|
|
--no-status \
|
|
--output=failures \
|
|
--file-output=sql/ascii/links.sql \
|
|
--config="$output_dir/linkcheckerrc" \
|
|
$f |\
|
|
awk -F"', '" '{ print $2; }' |\
|
|
sed 's/'"'"')"$//' |\
|
|
sed 's/^/Failed: /' |\
|
|
sed 's/$/; will try again at the end of the run./';
|
|
|
|
# Print the number of links we checked and the number we filtered.
|
|
total_links=`cat links.sql | grep -v '^--' | grep 'http' | wc -l`;
|
|
filtered_links=`grep 'filtered' links.sql | grep -v '^--' | grep 'http' |\
|
|
wc -l`;
|
|
echo " $filtered_links of $total_links external links were cached.";
|
|
|
|
# Insert results into the database. We have to insert the timestamp and the
|
|
# number of days the result is valid for. For that, we use a random number
|
|
# of days, because we don't want *all* of our results to expire on the same
|
|
# CI run and have it take forever.
|
|
cat links.sql |\
|
|
sed 's/modified) values (/modified,resulttime,validdays) values (/' |\
|
|
sed "s/);$/, current_timestamp, random() % 10 + 25);/" |\
|
|
sqlite3 "$output_dir/all_links.db";
|
|
|
|
# Print any warnings too, because we will try them again later.
|
|
cat create.sql | sqlite3 tmp.db;
|
|
cat links.sql |\
|
|
sed 's/modified) values (/modified,resulttime,validdays) values (/' |\
|
|
sed "s/);$/, current_timestamp, random() % 10 + 25);/" |\
|
|
sqlite3 tmp.db;
|
|
echo "SELECT DISTINCT urlname, warning FROM linksdb
|
|
WHERE valid = 1 AND
|
|
warning IS NOT NULL AND
|
|
(result NOT LIKE '200%' AND
|
|
warning NOT LIKE '%307 Temporary Redirect%' AND
|
|
result <> 'filtered' AND
|
|
result <> 'syntax OK');" |\
|
|
sqlite3 tmp.db |\
|
|
awk -F'|' '{ print "Warning: "$1": "$2"; will try again at the end of the run."; }';
|
|
rm -f tmp.db;
|
|
done
|
|
|
|
# Second chance on errors and warnings: filter out any spurious failures.
|
|
echo "SELECT DISTINCT urlname FROM linksdb
|
|
WHERE valid = 0 OR
|
|
(warning IS NOT NULL AND
|
|
warning NOT LIKE '%307 Temporary Redirect%') OR
|
|
(result NOT LIKE '200%' AND
|
|
result <> 'filtered' AND
|
|
result <> 'syntax OK');" | sqlite3 "$output_dir/all_links.db" >\
|
|
links_to_check.txt;
|
|
num_links=`cat links_to_check.txt | wc -l`;
|
|
if [ $num_links -gt 0 ];
|
|
then
|
|
echo "Second check for the following URLs that failed the first time:";
|
|
cat links_to_check.txt | sed 's/^/ /';
|
|
|
|
# Slow down the process to try and fix any links that got rate limited.
|
|
cat "$output_dir/linkcheckerrc.in" |\
|
|
sed 's/maxrequestspersecond=.*$/maxrequestspersecond=1/' >\
|
|
"$output_dir/linkcheckerrc";
|
|
|
|
linkchecker --check-extern \
|
|
--recursion-level=0 \
|
|
--threads=1 \
|
|
--file-output=sql/ascii/links_failed.sql \
|
|
--no-status \
|
|
--verbose \
|
|
--config="$output_dir/linkcheckerrc" \
|
|
`cat links_to_check.txt | tr '\n' ' '`;
|
|
|
|
cat create.sql | sqlite3 tmp.db;
|
|
cat links_failed.sql |\
|
|
sed 's/modified) values (/modified,resulttime,validdays) values (/' |\
|
|
sed "s/);$/, current_timestamp, random() % 10 + 25);/" |\
|
|
sqlite3 tmp.db;
|
|
echo "SELECT DISTINCT urlname, result FROM linksdb
|
|
WHERE valid = 0" | sqlite3 tmp.db |\
|
|
awk -F'|' '{ print " "$1": "$2; }' > links_failed.txt;
|
|
echo "SELECT DISTINCT urlname, warning FROM linksdb
|
|
WHERE valid = 1 AND warning IS NOT NULL" | sqlite3 tmp.db |\
|
|
awk -F'|' '{ print " "$1": "$2; }' > links_warned.txt;
|
|
|
|
# Also add the second pass results to the global cache.
|
|
cat links_failed.sql |\
|
|
sed 's/modified) values (/modified,resulttime,validdays) values (/' |\
|
|
sed "s/);$/, current_timestamp, random() % 10 + 25);/" |\
|
|
sqlite3 "$output_dir/all_links.db";
|
|
|
|
total_links_failed=`cat links_failed.txt links_warned.txt | wc -l`;
|
|
if [ $total_links_failed -gt 0 ];
|
|
then
|
|
echo "The following links have failed:";
|
|
|
|
cat links_failed.txt links_warned.txt;
|
|
rm -f links_failed.sql tmp.db links_failed.txt links_warned.txt;
|
|
exitcode=1;
|
|
else
|
|
exitcode=0;
|
|
fi
|
|
|
|
rm -f tmp.db links_failed.sql;
|
|
else
|
|
exitcode=0;
|
|
fi
|
|
|
|
rm -f links_to_check.txt;
|
|
|
|
# Add to the global cache.
|
|
if [ ! -z ${LINK_CACHE_FILE+x} ];
|
|
then
|
|
mv "$output_dir/all_links.db" "${LINK_CACHE_FILE}";
|
|
echo "DELETE FROM linksdb
|
|
WHERE valid = 0 OR
|
|
warning IS NOT NULL OR
|
|
julianday(datetime()) - julianday(resulttime) >= validdays;" |\
|
|
sqlite3 "${LINK_CACHE_FILE}";
|
|
|
|
# Keep only the most recent entry for a given urlname, to keep the size of
|
|
# the cache as small as possible.
|
|
echo "CREATE TABLE tmp_linksdb AS SELECT * FROM linksdb
|
|
GROUP BY urlname HAVING MAX(resulttime) ORDER BY urlname;" |\
|
|
sqlite3 "${LINK_CACHE_FILE}";
|
|
echo "DROP TABLE linksdb;" | sqlite3 "${LINK_CACHE_FILE}";
|
|
echo "ALTER TABLE tmp_linksdb RENAME TO linksdb;" | sqlite3 "${LINK_CACHE_FILE}";
|
|
fi
|
|
|
|
# Pick all the links that are within a week of timing out and run them again,
|
|
# to see if we can "refresh" them. This is intended to handle situations
|
|
# where flaky URLs may not always work, but they will be tried a handful of
|
|
# times over the week before their last run expires. The hope is that one of
|
|
# those runs in the last week before they expire will succeed, preventing a
|
|
# documentation job from failing due to a bad link.
|
|
echo "SELECT DISTINCT urlname FROM linksdb
|
|
WHERE valid = 1 AND
|
|
urlname LIKE 'http%' AND
|
|
validdays -
|
|
(julianday(datetime()) - julianday(resulttime)) <= 7 AND
|
|
(result LIKE '200%' OR
|
|
result = 'filtered' OR
|
|
result = 'syntax OK');" |\
|
|
sqlite3 "$output_dir/all_links.db" > links_to_check.txt;
|
|
num_links=`cat links_to_check.txt | wc -l`;
|
|
if [ $num_links -gt 0 ];
|
|
then
|
|
echo "Checking $num_links links before their cache entry expires...";
|
|
linkchecker --check-extern \
|
|
--recursion-level=0 \
|
|
--threads=1 \
|
|
--file-output=sql/ascii/links_output.sql \
|
|
--output=failures \
|
|
--no-status \
|
|
--verbose \
|
|
--config="$output_dir/linkcheckerrc" \
|
|
`cat links_to_check.txt | tr '\n' ' '` |\
|
|
awk -F"', '" '{ print $2; }' |\
|
|
sed 's/'"'"')"$//' |\
|
|
sed 's/^/Warning: /' |\
|
|
sed 's/$/ failed, but cache entry not yet expired./';
|
|
|
|
cat links_output.sql |\
|
|
sed 's/modified) values (/modified,resulttime,validdays) values (/' |\
|
|
sed "s/);$/, current_timestamp, random() % 10 + 25);/" |\
|
|
sqlite3 "$output_dir/all_links.db";
|
|
# Filter out any bad links.
|
|
echo "DELETE FROM all_links WHERE valid = 0;" |\
|
|
sqlite3 "$output_dir/all_links.db";
|
|
fi
|
|
|
|
# Clean up unnecessary files.
|
|
rm -f "$output_dir/link_errors.csv" "$output_dir/all_links.csv" \
|
|
"$output_dir/linkcheckerrc.in" "$output_dir/linkcheckerrc";
|
|
rm -f links.csv links_failed.csv;
|
|
else
|
|
exitcode=0;
|
|
fi
|
|
|
|
# Remove temporary files.
|
|
rm -f create.sql;
|
|
if [ "a$del_header" == "a1" ];
|
|
then
|
|
rm -f "$template_html_header";
|
|
fi
|
|
|
|
if [ "a$del_footer" == "a1" ];
|
|
then
|
|
rm -f "$template_html_footer";
|
|
fi
|
|
|
|
exit $exitcode;
|