Ryan Curtin
c8eb3dceac
* Add pipeline to documentation homepage. * Fix for mobile devices. * Add little pipelines to go at the top of each page. * Overhaul index page. * Overhaul README to remove redundant material. * Add installation documentation. * Update pipelines. * Allow nesting of deeper details. * Add a pipeline to the top of the load/save page. * Add prerequisites link to main pipeline. * Add better but not finished sidebar. * Add a couple new documentation pages. * Fix URLs in svg. * Incremental checkin. * Fix Youtube URLs. * Incremental checkin. * Minor fixes. * Add first pass at evaluation/deployment pages. * Minor spacing and link fixes. * Flesh out a number of additional pages and write basic compilation documentation. * Fix some minor issues, and add Docker deployment page (not totally finished yet). * Add developer documentation landing page. * Hopefully getting close to the final set of changes here. * Remove this documentation for now. * Fix a few links, and the size of the sidebar. * Fix some additional links. * Fix a bunch more links. * Fix another link that now has a better place. * Refactor test-docs.sh to handle documentation that is a standalone program. * Fix file exclusions. * Fully qualify typename. * Update name of file. * Fix syntax error. * Remove files that are not meant to be compiled. * Also skip the quickstart. * Move quickstart entry to the top. * Remove gray coloring of binding documentation. * Update name of sidebar link. * Update to working link. * Fix Wikipedia anchor.
408 lines
11 KiB
Bash
Executable File
408 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Extract C++ code blocks from either an individual Markdown file or a directory
|
|
# full of Markdown files. This does roughly what you would expect it to, but
|
|
# there is a little bit of magic:
|
|
#
|
|
# * All ```c++ code blocks are extracted into their own .cpp files.
|
|
#
|
|
# * mlpack.hpp is included in each file, and each code block is placed in an
|
|
# `int main() { }` block.
|
|
#
|
|
# * If Eigen or xtensor is detected, correct includes are added.
|
|
#
|
|
# * All data referenced in http URLs is downloaded.
|
|
#
|
|
# * If a code block is just a class declaration, it will be added to the *next*
|
|
# code block file.
|
|
#
|
|
# Once all code blocks are compiled, they are run just to make sure they run
|
|
# correctly. If a single file was passed, all inputs and outputs are printed;
|
|
# if an entire directory is given, program output is only printed on error.
|
|
|
|
if [[ $# -ne 1 ]];
|
|
then
|
|
echo "Usages: " >&2;
|
|
echo " - $0 input_file.md" >&2;
|
|
echo " - $0 markdown_directory/" >&2;
|
|
exit 2;
|
|
fi
|
|
|
|
if [ -z "$CXX" ];
|
|
then
|
|
echo "You must set \$CXX to the compiler you want to use!" >&2;
|
|
exit 1;
|
|
fi
|
|
|
|
if [ -z "$CXXFLAGS" ];
|
|
then
|
|
echo "Warning: \$CXXFLAGS is unset. If ensmallen, Armadillo, STB, or cereal ";
|
|
echo " are not in standard locations, builds will fail! Be sure to use ";
|
|
echo " absolute paths, not relative paths.";
|
|
fi
|
|
|
|
if [ -z "$LDFLAGS" ];
|
|
then
|
|
echo "Warning: \$LDFLAGS is unset. If libarmadillo.so is not in a standard ";
|
|
echo " location, builds will fail!";
|
|
fi
|
|
|
|
# First determine what files we are looking at.
|
|
if [ -d $1 ];
|
|
then
|
|
files=`find $1 -iname '*.md'`;
|
|
mode="directory";
|
|
else
|
|
files=$1;
|
|
mode="file";
|
|
fi
|
|
|
|
# Extract the C++ code blocks from a particular file, creating
|
|
# $output_prefix1.cpp, $output_prefix2.cpp, and so on and so forth.
|
|
#
|
|
# The code in those snippets will be placed into an int main() { } block, and
|
|
# mlpack.hpp will be included.
|
|
extract_code_blocks()
|
|
{
|
|
input_file=$1;
|
|
output_prefix=$2;
|
|
|
|
# Extract into temporary files.
|
|
sed -n '/^```c++/,/^```/ p' < $input_file > $input_file.tmp;
|
|
output_file_id=0;
|
|
output_file_display="00"; # Hopefully no file has more than 100 examples...
|
|
|
|
# Track whether or not the last line was a fence, since we get them two at a
|
|
# time. We initially set this to 1, because the first fence does not have a
|
|
# preceding fence close above it.
|
|
last_line_fence=1;
|
|
|
|
while IFS= read -r line;
|
|
do
|
|
if [[ $last_line_fence == 1 ]];
|
|
then
|
|
# Skip this line---it will be a fence opening.
|
|
last_line_fence=0;
|
|
continue;
|
|
fi
|
|
|
|
if [[ $line == '```' ]];
|
|
then
|
|
last_line_fence=1;
|
|
|
|
if [ -f $output_prefix$output_file_display.body.cpp ];
|
|
then
|
|
# Determine whether we need a main() function for the code. Also check
|
|
# whether the file is simply a class definition, in which case we don't
|
|
# need to do anything except prepare it to be inserted into the next
|
|
# example.
|
|
has_main=`grep 'int main(' $output_prefix$output_file_display.body.cpp | wc -l`;
|
|
has_class1=`grep '^ class\|^ struct' $output_prefix$output_file_display.body.cpp | wc -l`;
|
|
has_class2=`grep '^ };' $output_prefix$output_file_display.body.cpp | wc -l`;
|
|
class_decl=0;
|
|
if [ $has_class1 -ne 0 -a $has_class2 -ne 0 ];
|
|
then
|
|
class_decl=1;
|
|
fi;
|
|
|
|
if [ $has_main -eq 0 -a $class_decl -eq 0 ];
|
|
then
|
|
# Create main() function to wrap the code in.
|
|
echo "#include <mlpack.hpp>" > $output_prefix$output_file_display.cpp;
|
|
echo "" >> $output_prefix$output_file_display.cpp;
|
|
|
|
# Insert any class definitions.
|
|
if [ -f $output_prefix$output_file_display.defn.cpp ];
|
|
then
|
|
cat $output_prefix$output_file_display.defn.cpp >> $output_prefix$output_file_display.cpp;
|
|
rm -f $output_prefix$output_file_display.defn.cpp;
|
|
fi
|
|
|
|
echo "int main()" >> $output_prefix$output_file_display.cpp;
|
|
echo "{" >> $output_prefix$output_file_display.cpp;
|
|
|
|
# Insert the code itself.
|
|
cat $output_prefix$output_file_display.body.cpp >> $output_prefix$output_file_display.cpp;
|
|
rm -f $output_prefix$output_file_display.body.cpp;
|
|
|
|
# Close main() function.
|
|
echo "}" >> $output_prefix$output_file_display.cpp;
|
|
elif [[ "$class_decl" == "1" ]];
|
|
then
|
|
# If the function is only a class declaration, set it aside, along
|
|
# with any other declarations, for the next program.
|
|
next_id=$(($output_file_id + 1));
|
|
next_display=$(printf "%02d" $next_id);
|
|
if [ -f $output_prefix$output_file_display.defn.cpp ];
|
|
then
|
|
mv $output_prefix$output_file_display.defn.cpp $output_prefix$next_display.defn.cpp;
|
|
cat $output_prefix$output_file_display.body.cpp >> $output_prefix$next_display.defn.cpp;
|
|
rm -f $output_prefix$output_file_display.body.cpp;
|
|
else
|
|
mv $output_prefix$output_file_display.body.cpp $output_prefix$next_display.defn.cpp;
|
|
fi
|
|
else
|
|
# The file should be able to compile on its own.
|
|
mv $output_prefix$output_file_display.body.cpp $output_prefix$output_file_display.cpp;
|
|
fi
|
|
|
|
# Detect if we need any to add any special headers. We have to do this
|
|
# when we finish with the file...
|
|
if [ -f $output_prefix$output_file_display.cpp ];
|
|
then
|
|
if [[ `grep 'Eigen::' $output_prefix$output_file_display.cpp | wc -l` -gt 0 ]];
|
|
then
|
|
sed -i '1s/^/#include <Eigen\/Dense>\n/' $output_prefix$output_file_display.cpp;
|
|
fi
|
|
|
|
if [[ `grep 'xt::' $output_prefix$output_file_display.cpp | wc -l` -gt 0 ]];
|
|
then
|
|
sed -i '1s/^/#include <xtensor\/xrandom.hpp>\n/' $output_prefix$output_file_display.cpp;
|
|
sed -i '1s/^/#include <xtensor\/xarray.hpp>\n/' $output_prefix$output_file_display.cpp;
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
output_file_id=$(($output_file_id + 1));
|
|
output_file_display=$(printf "%02d" $output_file_id);
|
|
|
|
continue;
|
|
fi
|
|
|
|
# Include indentation (two spaces).
|
|
echo " $line" >> $output_prefix$output_file_display.body.cpp;
|
|
done < $input_file.tmp;
|
|
|
|
rm -f $output_prefix*.defn.cpp; # Remove any unused definitions.
|
|
rm -f $input_file.tmp;
|
|
}
|
|
|
|
compile_code_blocks()
|
|
{
|
|
input_dir=$1;
|
|
|
|
# If there are no files to compile, leave early.
|
|
if ! compgen -G $input_dir/*.cpp >/dev/null;
|
|
then
|
|
return;
|
|
fi
|
|
|
|
for f in $input_dir/*.cpp;
|
|
do
|
|
echo " Compiling $f...";
|
|
of=${f%.cpp};
|
|
|
|
if ! $CXX -std=c++17 -Isrc/ $CXXFLAGS -o $of $f $LDFLAGS -larmadillo 2>$of.tmp;
|
|
then
|
|
echo "Compilation of the following program failed:";
|
|
echo "";
|
|
cat $f;
|
|
echo "";
|
|
echo "First ten lines of error output:";
|
|
head $of.tmp;
|
|
echo "";
|
|
echo "For full error output run either:";
|
|
echo " - less $of.tmp";
|
|
echo " - $CXX -std=c++17 -Isrc/ $CXXFLAGS -o $of $f $LDFLAGS -larmadillo";
|
|
echo "";
|
|
echo "Did you set \$CXX, \$CXXFLAGS, and \$LDFLAGS correctly?"
|
|
exit 1;
|
|
fi
|
|
done
|
|
}
|
|
|
|
download_http_artifacts()
|
|
{
|
|
input_dir=$1;
|
|
output_dir=$2;
|
|
|
|
# Get a list of all HTTP resources.
|
|
artifacts=`grep 'http[s]*://' $input_dir/*.cpp |\
|
|
sed 's/^.*\(http[^ ]*\).*$/\1/' |\
|
|
sort |\
|
|
uniq |\
|
|
grep 'csv\|arff\|bin\|png' |\
|
|
sed 's/\.$//'`;
|
|
cd $output_dir;
|
|
for a in $artifacts;
|
|
do
|
|
out_a=`basename $a`;
|
|
if [ ! -f $out_a ];
|
|
then
|
|
echo " Downloading $a...";
|
|
if ! curl -s -O $a;
|
|
then
|
|
echo "Error downloading $a!";
|
|
exit 1;
|
|
fi
|
|
fi
|
|
done
|
|
cd - >/dev/null;
|
|
|
|
# Special case: if we are looking at core.md, this has two special files we
|
|
# need to create that is used in the example.
|
|
f=`basename $input_dir`;
|
|
if [[ "$f" == "core" || "$f" == "matrices" ]];
|
|
then
|
|
cd $output_dir;
|
|
echo " Creating data.csv...";
|
|
cat > data.csv << EOF
|
|
3,3,3,3,0
|
|
3,4,4,3,0
|
|
3,4,4,3,0
|
|
3,3,4,3,0
|
|
3,6,4,3,0
|
|
2,4,4,3,0
|
|
2,4,4,1,0
|
|
3,3,3,2,0
|
|
3,4,4,2,0
|
|
3,4,4,2,0
|
|
3,3,4,2,0
|
|
3,6,4,2,0
|
|
2,4,4,2,0
|
|
EOF
|
|
|
|
echo " Creating mixed_string_data.csv...";
|
|
cat > mixed_string_data.csv << EOF
|
|
3,"hello",3,"f",0
|
|
3,"goodbye",4,"f",0
|
|
3,"goodbye",4,"e",0
|
|
3,"hello",4,"d",0
|
|
3,"hello",4,"d",0
|
|
2,"hello",4,"d",0
|
|
2,"hello",4,"d",0
|
|
3,"goodbye",3,"f",0
|
|
3,"goodbye",4,"f",0
|
|
3,"hello",4,"f",0
|
|
3,"hello",4,"c",0
|
|
3,"hello",4,"f",0
|
|
2,"hello",4,"c",0
|
|
EOF
|
|
cd - >/dev/null;
|
|
fi
|
|
}
|
|
|
|
run_code_blocks()
|
|
{
|
|
input_dir=$1;
|
|
|
|
for f in $input_dir/*.cpp;
|
|
do
|
|
f_exec=${f%.cpp};
|
|
if [[ "$mode" == "directory" ]];
|
|
then
|
|
echo " Running $f_exec...";
|
|
if ! ./$f_exec 2>&1 >/dev/null;
|
|
then
|
|
echo " Error running $f_exec!";
|
|
echo " --------------------------------------------------------------------- ";
|
|
echo " Contents of $f:";
|
|
echo "";
|
|
cat $f;
|
|
echo "";
|
|
exit 1;
|
|
fi
|
|
else
|
|
echo " --------------------------------------------------------------------- ";
|
|
echo " Contents of $f:";
|
|
echo "";
|
|
cat $f;
|
|
echo "";
|
|
echo " --------------------------------------------------------------------- ";
|
|
echo " Output of $f_exec:";
|
|
echo "";
|
|
|
|
if ! ./$f_exec;
|
|
then
|
|
echo "";
|
|
echo "Error running $f_exec! See output above.";
|
|
exit 1;
|
|
fi
|
|
echo "";
|
|
echo " --------------------------------------------------------------------- ";
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Main loop: process the files we were asked to process.
|
|
mkdir -p doc/build/;
|
|
for f in $files;
|
|
do
|
|
if [[ "$mode" == "directory" ]];
|
|
then
|
|
declare -a files_to_skip=(
|
|
# These files have small incomplete snippets that can't compile into
|
|
# standalone programs.
|
|
"deploy_windows.md"
|
|
"hpt.md"
|
|
"cv.md"
|
|
"timer.md"
|
|
"bindings.md"
|
|
"iodoc.md"
|
|
"distances.md"
|
|
"elemtype.md"
|
|
"kernels.md"
|
|
"trees.md"
|
|
# Skip the quickstart, since it depends on some specific data.
|
|
"cpp.md"
|
|
# The tutorials are old and are likely to be replaced, so let's not test
|
|
# them.
|
|
"amf.md"
|
|
"ann.md"
|
|
"approx_kfn.md"
|
|
"cf.md"
|
|
"datasetmapper.md"
|
|
"det.md"
|
|
"emst.md"
|
|
"fastmks.md"
|
|
"image.md"
|
|
"kmeans.md"
|
|
"linear_regression.md"
|
|
"neighbor_search.md"
|
|
"range_search.md"
|
|
"reinforcement_learning.md"
|
|
"asynchronous_learning.md"
|
|
"ddpg.md"
|
|
"q_learning.md"
|
|
"sac.md"
|
|
"td3.md"
|
|
);
|
|
|
|
skip=0;
|
|
for skip_f in "${files_to_skip[@]}";
|
|
do
|
|
base_f=`basename $f`;
|
|
if [ "$base_f" = "$skip_f" ];
|
|
then
|
|
skip=1;
|
|
break;
|
|
fi
|
|
done
|
|
|
|
if [[ $skip -eq 1 ]];
|
|
then
|
|
continue;
|
|
fi
|
|
fi
|
|
|
|
echo "Building documentation for $f...";
|
|
|
|
build_dir_tmp=${f#doc/};
|
|
build_dir=${build_dir_tmp%.md};
|
|
base_file=`basename $f .md`;
|
|
mkdir -p doc/build/$build_dir/;
|
|
|
|
extract_code_blocks $f doc/build/$build_dir/$base_file;
|
|
# If there are no C++ files, don't do anything else..
|
|
if ! compgen -G doc/build/$build_dir/*.cpp >/dev/null;
|
|
then
|
|
continue;
|
|
fi
|
|
|
|
compile_code_blocks doc/build/$build_dir;
|
|
download_http_artifacts doc/build/$build_dir doc/build/;
|
|
cd doc/build/;
|
|
run_code_blocks $build_dir;
|
|
cd ../../;
|
|
done
|