OLD IDEA / no longer active: New modular plugin design + 3 new plugins: browsertrix-crawler, ReplayWeb.page, gallery-dl

Dockerfile

@@ -113,9 +113,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
  # 1. packaging dependencies
  apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
  # 2. docker and init system dependencies
- zlib1g-dev dumb-init gosu cron unzip grep \
+ zlib1g-dev dumb-init gosu cron unzip grep ncat \
  # 3. frivolous CLI helpers to make debugging failed archiving easier
- # nano iputils-ping dnsutils htop procps jq yq
+ # nano iputils-ping dnsutils htop procps jq yq \
  && rm -rf /var/lib/apt/lists/*
 
 ######### Language Environments ####################################

archivebox/core/settings.py

@@ -61,6 +61,12 @@
  'django.contrib.admin',
 
  'core',
+
+ # Plugins
+ 'plugins.replaywebpage',
+ # ...
+ # someday we may have enough plugins to justify dynamic loading:
+ # *(path.parent.name for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/apps.py')),,
 
  'django_extensions',
 ]
@@ -162,7 +168,7 @@
  'debug_toolbar.panels.request.RequestPanel',
  'debug_toolbar.panels.sql.SQLPanel',
  'debug_toolbar.panels.staticfiles.StaticFilesPanel',
- # 'debug_toolbar.panels.templates.TemplatesPanel',
+ # 'debug_toolbar.panels.templates.TemplatesPanel', # buggy/slow
  'debug_toolbar.panels.cache.CachePanel',
  'debug_toolbar.panels.signals.SignalsPanel',
  'debug_toolbar.panels.logging.LoggingPanel',
@@ -178,16 +184,35 @@
 
 STATIC_URL = '/static/'
 
+STATIC_ROOT = Path(PACKAGE_DIR) / 'collected_static'
+
 STATICFILES_DIRS = [
  *([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
  str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
+
+ # Plugins
+ str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/static'),
+ # ...
+ # someday if there are many more plugins / user-addable plugins:
+ # *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/static')),
 ]
 
+MEDIA_URL = '/archive/'
+MEDIA_ROOT = OUTPUT_DIR / 'archive'
+
+
 TEMPLATE_DIRS = [
  *([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
  str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
  str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
  str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
+
+ # Plugins
+ str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/templates')
+ # ...
+ #
+ # someday if there are many more plugins / user-addable plugins:
+ # *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/templates')),
 ]
 
 TEMPLATES = [

archivebox/core/urls.py

@@ -8,6 +8,7 @@
 
 from core.views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView
 
+
 # GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
 # from config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
 # GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
@@ -26,6 +27,9 @@
  path('archive/', RedirectView.as_view(url='/')),
  path('archive/<path:path>', SnapshotView.as_view(), name='Snapshot'),
 
+ path('plugins/replaywebpage/', include('plugins.replaywebpage.urls')),
+ # ... dynamic load these someday if there are more of them
+
  path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
  path('add/', AddView.as_view(), name='add'),
 

archivebox/manage.py

@@ -7,7 +7,7 @@
  # versions of ./manage.py commands whenever possible. When that's not possible
  # (e.g. makemigrations), you can comment out this check temporarily
 
- if not ('makemigrations' in sys.argv or 'migrate' in sys.argv):
+ if not ('makemigrations' in sys.argv or 'migrate' in sys.argv or 'collectstatic' in sys.argv):
  print("[X] Don't run ./manage.py directly (unless you are a developer running makemigrations):")
  print()
  print(' Hint: Use these archivebox CLI commands instead of the ./manage.py equivalents:')

archivebox/plugins/__init__.py

@@ -0,0 +1,3 @@
+__package__ = 'archivebox.plugins'
+
+

archivebox/plugins/replaywebpage/__init__.py

@@ -0,0 +1 @@
+__package__ = 'archivebox.plugins.replaywebpage'

archivebox/plugins/replaywebpage/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class ReplayWebPageConfig(AppConfig):
+ label = "ReplayWeb.Page"
+ name = "plugin_replaywebpage"
+
+ default_auto_field = "django.db.models.BigAutoField"

archivebox/plugins/replaywebpage/extractors.py

@@ -0,0 +1,50 @@
+# browsertrix extractor
+
+def save_browsertrix(link, out_dir, timeout, config):
+
+
+ browsertrix_dir = out_dir / 'browsertrix'
+ browsertrix_dir.mkdir(exist_ok=True)
+
+ crawl_id = link.timestamp
+
+ browsertrix_crawler_cmd = [
+ 'crawl',
+ f'--url', link.url,
+ f'--collection={crawl_id}',
+ '--scopeType=page',
+ '--generateWACZ',
+ '--text=final-to-warc',
+ '--timeLimit=60',
+ ]
+
+ remote_cmd = """
+ rm /tmp/dump.rdb;
+ rm -rf /crawls/collections;
+ mkdir /crawls/collections;
+ env CRAWL_ID={crawl_id} 
+ """
+
+ local_cmd = ['nc', 'browsertrix', '2222']
+
+ status = 'succeeded'
+ timer = TimedProgress(timeout, prefix=' ')
+ try:
+ result = run(local_cmd, cwd=str(out_dir), input=remote_cmd, timeout=timeout)
+
+ cmd_output = result.stdout.decode()
+
+ wacz_output_file = Path('/browsertrix/crawls') / crawl_id / f'{crawl_id}'.wacz
+
+ copy_and_overwrite(wacz_output_file, browsertrix_dir / wacz_output_file.name)
+
+
+
+TEMPLATE = """
+
+"""
+
+# rm /tmp/dump.rdb;
+# rm -rf /crawls/collections;
+# mkdir /crawls/collections;
+# env CRAWL_ID=tec2342 crawl --url 'https://example.com' --scopeType page --generateWACZ --collection tec2342 --text final-to-warc --timeLimit 60

archivebox/plugins/replaywebpage/static/test.txt

@@ -0,0 +1 @@
+test content this should be visible

archivebox/plugins/replaywebpage/templates/plugin_replaywebpage__viewer.html

@@ -0,0 +1,39 @@
+{% load tz core_tags static %}
+
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <title>{{title}}</title>
+ <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
+
+ </style>
+ <style>
+ html, body {
+ width: 100%;
+ height: 100%;
+ background-color: #ddd;
+ }
+
+ </style>
+ </head>
+ <body>
+ ReplayWeb.page for: {{snapshot.url}} ({{timestamp}}) /{{warc_filename}}
+
+ {{snapshot}}
+
+ <script>
+ // https://cdn.jsdelivr.net/npm/[email protected]/sw.min.js
+ // https://cdn.jsdelivr.net/npm/[email protected]/ui.min.js
+ </script>
+
+ <style>
+ replay-web-page {
+ width: 100%;
+ height: 900px;
+ }
+ </style>
+ <script src="/static/ui.js"></script>
+
+ <replay-web-page style="height: 600px" embed="replay-with-info" replayBase="/static/" source="{% static 'test.wacz' %}" url="https://example.com/"></replay-web-page>
+ </body>
+</html>

archivebox/plugins/replaywebpage/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from .views import ReplayWebPageViewer
+
+urlpatterns = [
+ path('<path:path>', ReplayWebPageViewer.as_view(), name='plugin_replaywebpage__viewer'),
+]

archivebox/plugins/replaywebpage/views.py

@@ -0,0 +1,47 @@
+import os
+import sys
+from pathlib import Path
+
+from django.views import View
+from django.shortcuts import render
+from django.db.models import Q
+
+from core.models import Snapshot
+
+# from archivebox.config import PUBLIC_SNAPSHOTS
+PUBLIC_SNAPSHOTS = True
+
+
+class ReplayWebPageViewer(View):
+ template_name = 'plugin_replaywebpage__viewer.html'
+
+ # render static html index from filesystem archive/<timestamp>/index.html
+
+ def get_context_data(self, **kwargs):
+ return {
+ # **super().get_context_data(**kwargs),
+ # 'VERSION': VERSION,
+ # 'COMMIT_HASH': COMMIT_HASH,
+ # 'FOOTER_INFO': FOOTER_INFO,
+ }
+
+
+ def get(self, request, path):
+ if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+ return redirect(f'/admin/login/?next={request.path}')
+
+ try:
+ timestamp, warc_filename = path.split('/', 1)
+ except (IndexError, ValueError):
+ timestamp, warc_filename = path.split('/', 1)[0], ''
+
+ snapshot = Snapshot.objects.get(Q(timestamp=timestamp) | Q(id__startswith=timestamp))
+
+ context = self.get_context_data()
+ context.update({
+ "snapshot": snapshot,
+ "timestamp": timestamp,
+ "warc_filename": warc_filename,
+ })
+ return render(template_name=self.template_name, request=self.request, context=context)
+

bin/docker_ipc_listener.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+# Allow another docker container to run commands on this container
+# This is the script to run on the server container.
+# The client can connect and run a command like so:
+# $ echo whoami | nc servercontainername 2222
+# root
+
+import socket
+import subprocess as sp
+from datetime import datetime
+
+LISTEN_PORT = 2222
+
+s1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+s1.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+s1.bind(("0.0.0.0", LISTEN_PORT))
+s1.listen(1)
+print("Listening for shell commands on 0.0.0.0:2222", flush=True)
+
+conn, addr = s1.accept()
+while True:
+ cmd = conn.recv(1024).decode()
+ if not cmd:
+ conn, addr = s1.accept()
+ continue
+
+ timestamp = datetime.now().isoformat()
+ client_ip, client_port = conn.getsockname()
+ print(f'\n[{timestamp}][{client_ip}:{client_port}] $', cmd)
+
+ with sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.STDOUT, stdin=sp.PIPE, bufsize=1, universal_newlines=True) as p:
+ for line in p.stdout:
+ print(line.strip(), flush=True)
+ conn.sendall(line.encode("utf-8"))
+
+ conn.close()
+ conn, addr = s1.accept()

docker-compose.yml

@@ -13,12 +13,14 @@ version: '3.9'
 services:
  archivebox:
  #image: ${DOCKER_IMAGE:-archivebox/archivebox:dev}
- image: archivebox/archivebox:dev
+ image: archivebox:test
+ # image: archivebox/archivebox:dev
  command: server --quick-init 0.0.0.0:8000
  ports:
  - 8000:8000
  volumes:
  - ./data:/data
+ - /Volumes/OPT/browsertrix:/browsertrix:z
  # - ./etc/crontabs:/var/spool/cron/crontabs # uncomment this and archivebox_scheduler below to set up automatic recurring archive jobs
  # - ./archivebox:/app/archivebox # uncomment this to mount the ArchiveBox source code at runtime (for developers working on archivebox)
  # build: . # uncomment this to build the image from source code at buildtime (for developers working on archivebox)
@@ -48,6 +50,14 @@ services:
  # dns:
  # - 172.20.0.53
 
+ browsertrix:
+ image: webrecorder/browsertrix-crawler:latest
+ command: /bin/docker_ipc_listener.py
+ expose:
+ - 2222
+ volumes:
+ - /Volumes/OPT/browsertrix:/crawls:z
+ - ./bin/docker_ipc_listener.py:/bin/docker_ipc_listener.py
 
  ######## Optional Addons: tweak examples below as needed for your specific use case ########
 

etc/uwsgi.ini

@@ -7,7 +7,12 @@ wsgi-file = archivebox/core/wsgi.py
 processes = 4
 threads = 1
 stats = 127.0.0.1:9191
-static-map /static=./archivebox/templates/static
+static-map = /static=./archivebox/templates/static
+static-map = /static=./archivebox/plugins/replaywebpage/static
+static-map = /archive=$(PWD)/archive
+static=index = index.html
 harakiri = 172800
 post-buffering = 1
 disable-logging = True
+check-static 
+honour-range = True