OLD IDEA / no longer active: New modular plugin design + 3 new plugins: browsertrix-crawler, ReplayWeb.page, gallery-dl

.gitignore

@@ -26,11 +26,9 @@ dist/
 
 # Data folders
 data/
-data1/
-data2/
-data3/
 data*/
 output/
 
 # vim
 *.sw?
+.vscode/

Dockerfile

@@ -15,8 +15,8 @@
 # Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
 
-# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
 FROM python:3.11-slim-bookworm
+# Uses Debian 12 w/ faster-updating apt-lists added below: https://packages.debian.org/bookworm-backports/
 
 LABEL name="archivebox" \
  maintainer="Nick Sweeting <[email protected]>" \
@@ -127,9 +127,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
  # 1. packaging dependencies
  apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
  # 2. docker and init system dependencies
- zlib1g-dev dumb-init gosu cron unzip grep \
+ zlib1g-dev dumb-init gosu cron unzip grep ncat \
  # 3. frivolous CLI helpers to make debugging failed archiving easier
- # nano iputils-ping dnsutils htop procps jq yq
+ # nano iputils-ping dnsutils htop procps jq yq \
  && rm -rf /var/lib/apt/lists/*
 
 ######### Language Environments ####################################

archivebox/config.py

@@ -38,7 +38,7 @@
 from pathlib import Path
 from datetime import datetime, timezone
 from typing import Optional, Type, Tuple, Dict, Union, List
-from subprocess import run, PIPE, DEVNULL
+from subprocess import run, PIPE, STDOUT, DEVNULL
 from configparser import ConfigParser
 from collections import defaultdict
 import importlib.metadata
@@ -854,7 +854,7 @@ def hint(text: Union[Tuple[str, ...], List[str], str], prefix=' ', config: Op
 
 
 # Dependency Metadata Helpers
-def bin_version(binary: Optional[str]) -> Optional[str]:
+def bin_version(binary: Optional[str], cmd=None) -> Optional[str]:
  """check the presence and return valid version line of a specified binary"""
 
  abspath = bin_path(binary)
@@ -863,11 +863,21 @@ def bin_version(binary: Optional[str]) -> Optional[str]:
 
  try:
  bin_env = os.environ | {'LANG': 'C'}
- version_str = run([abspath, "--version"], stdout=PIPE, env=bin_env).stdout.strip().decode()
+ is_cmd_str = cmd and isinstance(cmd, str)
+ version_str = run(cmd or [abspath, "--version"], shell=is_cmd_str, stdout=PIPE, stderr=STDOUT, env=bin_env).stdout.strip().decode()
  if not version_str:
- version_str = run([abspath, "--version"], stdout=PIPE).stdout.strip().decode()
- # take first 3 columns of first line of version info
- return ' '.join(version_str.split('\n')[0].strip().split()[:3])
+ version_str = run(cmd or [abspath, "--version"], shell=is_cmd_str, stdout=PIPE, stderr=STDOUT).stdout.strip().decode()
+
+ version_ptn = re.compile(r"\d+?\.\d+?\.?\d*", re.MULTILINE)
+ try:
+ version_nums = version_ptn.findall(version_str.split('\n')[0])[0]
+ if version_nums:
+ return version_nums
+ else:
+ raise IndexError
+ except IndexError:
+ # take first 3 columns of first line of version info
+ return ' '.join(version_str.split('\n')[0].strip().split()[:3])
  except OSError:
  pass
  # stderr(f'[X] Unable to find working version of dependency: {binary}', color='red')

archivebox/config_stubs.py

@@ -9,17 +9,19 @@
 SimpleConfigValueGetter = Callable[[], SimpleConfigValue]
 ConfigValue = Union[SimpleConfigValue, SimpleConfigValueDict, SimpleConfigValueGetter]
 
+SHArgs = List[str] # shell command args list e.g. ["--something=1", "--someotherarg"]
+
 
 class BaseConfig(TypedDict):
  pass
 
 class ConfigDict(BaseConfig, total=False):
  """
  # Regenerate by pasting this quine into `archivebox shell` 🥚
- from archivebox.config import ConfigDict, CONFIG_DEFAULTS
+ from archivebox.config import ConfigDict, CONFIG_SCHEMA
  print('class ConfigDict(BaseConfig, total=False):')
  print(' ' + '"'*3 + ConfigDict.__doc__ + '"'*3)
- for section, configs in CONFIG_DEFAULTS.items():
+ for section, configs in CONFIG_SCHEMA.items():
  for key, attrs in configs.items():
  Type, default = attrs['type'], attrs['default']
  if default is None:
@@ -32,24 +34,51 @@ class ConfigDict(BaseConfig, total=False):
  USE_COLOR: bool
  SHOW_PROGRESS: bool
  IN_DOCKER: bool
+ IN_QEMU: bool
+ PUID: int
+ PGID: int
 
- PACKAGE_DIR: Path
- OUTPUT_DIR: Path
- CONFIG_FILE: Path
+ OUTPUT_DIR: Optional[str]
+ CONFIG_FILE: Optional[str]
  ONLY_NEW: bool
  TIMEOUT: int
  MEDIA_TIMEOUT: int
  OUTPUT_PERMISSIONS: str
  RESTRICT_FILE_NAMES: str
  URL_DENYLIST: str
+ URL_ALLOWLIST: Optional[str]
+ ADMIN_USERNAME: Optional[str]
+ ADMIN_PASSWORD: Optional[str]
+ ENFORCE_ATOMIC_WRITES: bool
+ TAG_SEPARATOR_PATTERN: str
 
  SECRET_KEY: Optional[str]
  BIND_ADDR: str
  ALLOWED_HOSTS: str
  DEBUG: bool
  PUBLIC_INDEX: bool
  PUBLIC_SNAPSHOTS: bool
+ PUBLIC_ADD_VIEW: bool
  FOOTER_INFO: str
+ SNAPSHOTS_PER_PAGE: int
+ CUSTOM_TEMPLATES_DIR: Optional[str]
+ TIME_ZONE: str
+ TIMEZONE: str
+ REVERSE_PROXY_USER_HEADER: str
+ REVERSE_PROXY_WHITELIST: str
+ LOGOUT_REDIRECT_URL: str
+ PREVIEW_ORIGINALS: bool
+ LDAP: bool
+ LDAP_SERVER_URI: Optional[str]
+ LDAP_BIND_DN: Optional[str]
+ LDAP_BIND_PASSWORD: Optional[str]
+ LDAP_USER_BASE: Optional[str]
+ LDAP_USER_FILTER: Optional[str]
+ LDAP_USERNAME_ATTR: Optional[str]
+ LDAP_FIRSTNAME_ATTR: Optional[str]
+ LDAP_LASTNAME_ATTR: Optional[str]
+ LDAP_EMAIL_ATTR: Optional[str]
+ LDAP_CREATE_SUPERUSER: bool
 
  SAVE_TITLE: bool
  SAVE_FAVICON: bool
@@ -58,25 +87,50 @@ class ConfigDict(BaseConfig, total=False):
  SAVE_SINGLEFILE: bool
  SAVE_READABILITY: bool
  SAVE_MERCURY: bool
+ SAVE_HTMLTOTEXT: bool
  SAVE_PDF: bool
  SAVE_SCREENSHOT: bool
  SAVE_DOM: bool
+ SAVE_HEADERS: bool
  SAVE_WARC: bool
  SAVE_GIT: bool
  SAVE_MEDIA: bool
  SAVE_ARCHIVE_DOT_ORG: bool
+ SAVE_ALLOWLIST: dict
+ SAVE_DENYLIST: dict
 
  RESOLUTION: str
  GIT_DOMAINS: str
  CHECK_SSL_VALIDITY: bool
+ MEDIA_MAX_SIZE: str
  CURL_USER_AGENT: str
  WGET_USER_AGENT: str
  CHROME_USER_AGENT: str
- COOKIES_FILE: Union[str, Path, None]
- CHROME_USER_DATA_DIR: Union[str, Path, None]
+ COOKIES_FILE: Optional[str]
+ CHROME_USER_DATA_DIR: Optional[str]
  CHROME_TIMEOUT: int
  CHROME_HEADLESS: bool
  CHROME_SANDBOX: bool
+ YOUTUBEDL_ARGS: list
+ WGET_ARGS: list
+ CURL_ARGS: list
+ GIT_ARGS: list
+ SINGLEFILE_ARGS: Optional[list]
+ FAVICON_PROVIDER: str
+
+ USE_INDEXING_BACKEND: bool
+ USE_SEARCHING_BACKEND: bool
+ SEARCH_BACKEND_ENGINE: str
+ SEARCH_BACKEND_HOST_NAME: str
+ SEARCH_BACKEND_PORT: int
+ SEARCH_BACKEND_PASSWORD: str
+ SEARCH_PROCESS_HTML: bool
+ SONIC_COLLECTION: str
+ SONIC_BUCKET: str
+ SEARCH_BACKEND_TIMEOUT: int
+ FTS_SEPARATE_DATABASE: bool
+ FTS_TOKENIZERS: str
+ FTS_SQLITE_MAX_LENGTH: int
 
  USE_CURL: bool
  USE_WGET: bool
@@ -85,21 +139,22 @@ class ConfigDict(BaseConfig, total=False):
  USE_MERCURY: bool
  USE_GIT: bool
  USE_CHROME: bool
+ USE_NODE: bool
  USE_YOUTUBEDL: bool
+ USE_RIPGREP: bool
  CURL_BINARY: str
  GIT_BINARY: str
  WGET_BINARY: str
  SINGLEFILE_BINARY: str
  READABILITY_BINARY: str
  MERCURY_BINARY: str
  YOUTUBEDL_BINARY: str
+ NODE_BINARY: str
+ RIPGREP_BINARY: str
  CHROME_BINARY: Optional[str]
-
- YOUTUBEDL_ARGS: List[str]
- WGET_ARGS: List[str]
- CURL_ARGS: List[str]
- GIT_ARGS: List[str]
- TAG_SEPARATOR_PATTERN: str
+ POCKET_CONSUMER_KEY: Optional[str]
+ POCKET_ACCESS_TOKENS: dict
+ READWISE_READER_TOKENS: dict
 
 
 ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]

archivebox/core/__init__.py

@@ -1,2 +1,3 @@
 __package__ = 'archivebox.core'
 
+# default_app_config = 'core.apps.CoreAppConfig'

archivebox/core/admin.py

@@ -12,6 +12,7 @@
 from django.utils.safestring import mark_safe
 from django.shortcuts import render, redirect
 from django.contrib.auth import get_user_model
+from django.contrib.auth.models import Group, Permission
 from django import forms
 
 from ..util import htmldecode, urldecode, ansi_to_html
@@ -159,6 +160,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
  action_form = SnapshotActionForm
 
+
  def changelist_view(self, request, extra_context=None):
  extra_context = extra_context or {}
  return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)

archivebox/core/apps.py

@@ -1,9 +1,16 @@
 from django.apps import AppConfig
 
 
-class CoreConfig(AppConfig):
+class CoreAppConfig(AppConfig):
  name = 'core'
 
+ # label = 'Archive Data'
+ verbose_name = "Archive Data"
+
+ # WIP: broken by Django 3.1.2 -> 4.0 migration
+ # default_auto_field = 'django.db.models.UUIDField'
+
+
  def ready(self):
  from .auth import register_signals
 

archivebox/core/auth.py

@@ -1,5 +1,7 @@
-import os
-from django.conf import settings
+__package__ = 'archivebox.core'
+
+
+
 from ..config import (
  LDAP
 )

archivebox/core/models.py

@@ -50,7 +50,7 @@ class Tag(models.Model):
 
  class Meta:
  verbose_name = "Tag"
- verbose_name_plural = "Tags"
+ verbose_name_plural = "🏷️ Tags"
 
  def __str__(self):
  return self.name
@@ -98,6 +98,10 @@ class Snapshot(models.Model):
 
  keys = ('url', 'timestamp', 'title', 'tags', 'updated')
 
+ class Meta:
+ verbose_name = "Snapshot"
+ verbose_name_plural = "⭐️ Archived Webpages (Snapshots)"
+
  def __repr__(self) -> str:
  title = self.title or '-'
  return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
@@ -282,5 +286,9 @@ class ArchiveResult(models.Model):
 
  objects = ArchiveResultManager()
 
+ class Meta:
+ verbose_name = "ArchiveResult"
+ verbose_name_plural = "📑 Logs (ArchiveResults)"
+
  def __str__(self):
  return self.extractor