Update documentation configs

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
 current_version = 0.6.41
-files = grab/__init__.py
+files = grab/__init__.py docs/conf.py
 commit = True
 tag = True

.readthedocs.yaml

@@ -0,0 +1,13 @@
+version: 2
+formats: "all"
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.10"
+python:
+   install:
+   - requirements: "requirements.txt"
+   - requirements: "requirements_dev.txt"
+   - path: "."
+sphinx:
+   configuration: "docs/conf.py"

Makefile

@@ -1,4 +1,4 @@
-.PHONY: bootstrap venv deps dirs clean pytest test release mypy pylint flake8 bandit check build
+.PHONY: bootstrap venv deps dirs clean pytest test release mypy pylint flake8 bandit check build docs
 
 SHELL := /bin/bash
 FILES_CHECK_MYPY = grab
@@ -67,3 +67,7 @@ build:
 demo:
 	tox -e py3-demo \
 	&& tox -e py38-demo
+
+docs:
+	rm -rf docs/_build/html 
+	sphinx-build -j auto docs docs/_build/html

docs/api/grab_error.rst

@@ -19,8 +19,4 @@ Custom exception classes for Grab.
 .. autoclass:: GrabAuthError
 .. autoclass:: GrabTooManyRedirectsError
 .. autoclass:: GrabInvalidUrl
-
-
-
-.. module:: grab.util.warning
 .. autoclass:: GrabDeprecationWarning

docs/conf.py

@@ -1,260 +1,28 @@
-#
-# Grab documentation build configuration file, created by
-# sphinx-quickstart on Fri Mar 27 02:27:14 2015.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-# from unittest.mock import MagicMock
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-import os.path
-import sys
-from unittest.mock import MagicMock
-
-SOURCE_DIR = os.path.dirname(os.path.realpath(__file__))
-PROJECT_DIR = os.path.dirname(os.path.dirname(SOURCE_DIR))
-sys.path.insert(0, PROJECT_DIR)
-
-
-class Mock(MagicMock):
-    @classmethod
-    def __getattr__(cls, name):
-        return MagicMock()
-
-
-MOCK_MODULES = [
-    "selection",
-    "lxml",
-    "lxml.html",
-    "lxml.etree",
-]
-sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
-
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-# needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
 extensions = [
     "sphinx.ext.autodoc",
     "sphinx.ext.viewcode",
+    "autoapi.extension",
+    "myst_nb",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
 ]
 
-# Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
-
-# The suffix of source filenames.
 source_suffix = ".rst"
-
-# The encoding of source files.
-# source_encoding = 'utf-8-sig'
-
-# The master toctree document.
 master_doc = "index"
-
-# General information about the project.
 project = "Grab"
-copyright = "2015 – 2022, Gregory Petukhov"
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = "0.6"
-# The full version, including alpha/beta/rc tags.
-release = "0.6"
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-# language = None
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-# today = ''
-# Else, today_fmt is used as the format for a strftime call.
-# today_fmt = '%B %d, %Y'
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
+copyright = "2015 – 2023, Gregory Petukhov"
+version = "0.6.41"
+release = "0.6.41"
 exclude_patterns = []
-
-# The reST default role (used for this markup: `text`) to use for all
-# documents.
-# default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-# add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-# add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-# show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "sphinx"
-
-# A list of ignored prefixes for module index sorting.
-# modindex_common_prefix = []
-
-# If true, keep warnings as "system message" paragraphs in the built documents.
-# keep_warnings = False
-
-
-# -- Options for HTML output ----------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-# html_theme = 'sphinx_rtd_theme'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-# html_theme_options = {}
-
-# Add any paths that contain custom themes here, relative to this directory.
-html_theme_path = ["_templates"]
-
-# The name for this set of Sphinx documents.  If None, it defaults to
-# "<project> v<release> documentation".
-# html_title = None
-
-# A shorter title for the navigation bar.  Default is the same as html_title.
-# html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-# html_logo = None
-
-# The name of an image file (within the static path) to use as favicon of the
-# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-# html_favicon = None
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = []  #'_static']
-
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-# html_extra_path = []
-
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-# html_last_updated_fmt = '%b %d, %Y'
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-# html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-# html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-# html_additional_pages = {}
-
-# If false, no module index is generated.
-# html_domain_indices = True
-
-# If false, no index is generated.
-# html_use_index = True
-
-# If true, the index is split into individual pages for each letter.
-# html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-# html_show_sourcelink = True
-
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-# html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-# html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it.  The value of this option must be the
-# base URL from which the finished HTML is served.
-# html_use_opensearch = ''
-
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-# html_file_suffix = None
-
-# Output file base name for HTML help builder.
 htmlhelp_basename = "Grabdoc"
-
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #'papersize': 'letterpaper',
-    # The font size ('10pt', '11pt' or '12pt').
-    #'pointsize': '10pt',
-    # Additional stuff for the LaTeX preamble.
-    #'preamble': '',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
+latex_elements = {}
 latex_documents = [
     ("index", "Grab.tex", "Grab Documentation", "Gregory Petukhov", "manual"),
 ]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-# latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-# latex_use_parts = False
-
-# If true, show page references after internal links.
-# latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-# latex_show_urls = False
-
-# Documents to append as an appendix to all manuals.
-# latex_appendices = []
-
-# If false, no module index is generated.
-# latex_domain_indices = True
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
 man_pages = [("index", "grab", "Grab Documentation", ["Gregory Petukhov"], 1)]
-
-# If true, show URL addresses after external links.
-# man_show_urls = False
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
 texinfo_documents = [
     (
         "index",
@@ -266,15 +34,5 @@ def __getattr__(cls, name):
         "Miscellaneous",
     ),
 ]
-
-# Documents to append as an appendix to all manuals.
-# texinfo_appendices = []
-
-# If false, no module index is generated.
-# texinfo_domain_indices = True
-
-# How to display URL addresses: 'footnote', 'no', or 'inline'.
-# texinfo_show_urls = 'footnote'
-
-# If true, do not generate a @detailmenu in the "Top" node's menu.
-# texinfo_no_detailmenu = False
+html_theme = "sphinx_rtd_theme"
+autoapi_dirs = ["../grab"]  # location to parse for API reference

docs/index.rst

@@ -81,7 +81,6 @@ crawlers.
     spider/intro
     spider/task
     spider/task_queue
-    spider/cache
     spider/error_handling
     spider/transport
 
@@ -101,12 +100,7 @@ methods exist, what they do, what they return, and what parameters they accept.
 .. toctree::
     :maxdepth: 2
 
-    api/grab_base
-    api/grab_error
-    api/grab_cookie
-    api/grab_spider_base
-    api/grab_document
-    api/grab_spider_task
+    autoapi/index
 
 
 Indices and tables

requirements.txt

@@ -9,7 +9,10 @@
 #pylint
 #psutil
 #fastrq
-#sphinx
+sphinx
+myst-nb
+sphinx_rtd_theme
+sphinx-autoapi
 
 tox
 test_server>=0.0.42