feat(scripts): build index

- script to build the index
- github action to weekly check for new optech topics
- update README

.github/workflows/update-topics.yaml

@@ -0,0 +1,67 @@
+name: Update Topics Index (Weekly)
+
+on:
+  schedule:
+    - cron: '0 0 * * 0'  # Runs at 00:00 UTC every Sunday
+  workflow_dispatch:      # Allows manual triggering
+  push:
+    branches:
+      - 'test/*'         # Runs on any branch under test/
+
+# Add explicit permissions for the GITHUB_TOKEN
+permissions:
+  contents: write  # Allows pushing to the repository
+
+jobs:
+  update-topics:
+    runs-on: ubuntu-latest
+
+    # Add environment variables to control behavior
+    env:
+      IS_TEST: ${{ startsWith(github.ref, 'refs/heads/test/') }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: pip install -r scripts/requirements.txt
+
+      - name: Build Index
+        run: python scripts/build_index.py
+
+      - name: Check for changes
+        id: changes
+        run: |
+          git diff --quiet || echo "changes=true" >> $GITHUB_OUTPUT
+
+      - name: Configure Git
+        if: steps.changes.outputs.changes == 'true'
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "[email protected]"
+
+      # Add debug information in test mode
+      - name: Debug Info (Test Mode)
+        if: env.IS_TEST == 'true'
+        run: |
+          echo "Changes detected: ${{ steps.changes.outputs.changes }}"
+          git diff --stat
+          git status
+
+      - name: Commit and push if there are changes
+        if: steps.changes.outputs.changes == 'true'
+        run: |
+          git add topics_index.json TOPICS.md
+          # Add [TEST] prefix to commit message on test branches
+          if [[ "${{ env.IS_TEST }}" == "true" ]]; then
+            git commit -m "[TEST] Auto-update topics index"
+          else
+            git commit -m "Auto-update topics index"
+          fi
+          git push

README.md

@@ -1,2 +1,59 @@
-# topics-index
-A list of Bitcoin topics 
+# Bitcoin Topics Index
+
+An extensive index of Bitcoin-related topics, combining and enhancing the established [Bitcoin Optech Topics](https://bitcoinops.org/en/topics/) with additional relevant entries.
+
+See [TOPICS.md](TOPICS.md) for the categorized index or [topics_index.json](topics_index.json) for the machine-readable format.
+
+## Repository Structure
+
+```
+.
+├── topics/             # Bitcoin topics
+│   ├── topic1.yaml
+│   ├── topic2.yaml
+│   └── ...
+├── scripts/           # Build and maintenance scripts
+│   └── build_index.py
+├── topics_index.json  # topics index
+├── TOPICS.md         # Generated documentation
+└── README.md
+```
+
+## Topics Format
+
+Each topic is defined in YAML format with the following structure:
+
+```yaml
+title: "Topic Title" # Display name of the topic
+slug: "topic-slug" # URL-friendly identifier
+categories: # List of categories this topic belongs to
+  - "Category 1"
+  - "Category 2"
+aliases: # Optional: Alternative names for the topic
+  - "Alternative Name"
+  - "Another Name"
+excerpt: "A comprehensive description of the topic."
+```
+
+## Usage
+
+### Building the Index
+
+To build the combined index and documentation:
+
+```bash
+python scripts/build_index.py
+```
+
+This will:
+
+1. Fetch the latest topics from Bitcoin Optech's [/topics.json](https://bitcoinops.org/topics.json).
+2. Combine them with additional topics from the `topics/` directory
+3. Generate `topics_index.json` with the complete topics data
+4. Create `TOPICS.md` with categorized listings
+
+### Adding Topics
+
+1. Create a new YAML file in the `topics/` directory
+2. Follow the topics format described above
+3. Run the build script to update the index

scripts/build_index.py

@@ -0,0 +1,154 @@
+import requests
+import json
+import yaml
+import os
+import logging
+from typing import Dict, List
+from collections import defaultdict
+from pathlib import Path
+
+# Set up logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+class TopicsBuilder:
+    def __init__(self, optech_topics_url: str, topics_dir: str, root_dir: str):
+        self.optech_topics_url = optech_topics_url
+        self.topics_dir = topics_dir
+        self.root_dir = root_dir
+
+    def fetch_optech_topics(self) -> List[Dict]:
+        """Fetch topics directly from the Bitcoin Optech website."""
+        response = requests.get(self.optech_topics_url)
+        response.raise_for_status()
+        return response.json()
+
+    def load_yaml_file(self, filepath: str) -> Dict:
+        """Load a single YAML file."""
+        with open(filepath, "r") as f:
+            return yaml.safe_load(f)
+
+    def load_topics(self) -> List[Dict]:
+        """Load all YAML files from the topics directory."""
+        topics = []
+        if os.path.exists(self.topics_dir):
+            for filename in os.listdir(self.topics_dir):
+                if filename.endswith(".yaml"):
+                    filepath = os.path.join(self.topics_dir, filename)
+                    topic = self.load_yaml_file(filepath)
+                    topics.append(topic)
+        return topics
+
+    def build_topics(self) -> List[Dict]:
+        """Build complete topics list by combining Optech and additional topics."""
+        optech_topics = self.fetch_optech_topics()
+        additional_topics = self.load_topics()
+
+        # Create a dictionary of topics by slug for easy lookup
+        topic_dict = {topic["slug"]: topic for topic in optech_topics}
+
+        # Add or override with additional topics
+        for topic in additional_topics:
+            topic_dict[topic["slug"]] = topic
+
+        # Convert back to list and sort by title
+        combined_topics = list(topic_dict.values())
+        combined_topics.sort(key=lambda x: x["title"].lower())
+
+        return combined_topics
+
+    def write_topics_index(self, topics: List[Dict]):
+        """Write the topics index to a JSON file in the root directory."""
+        output_path = os.path.join(self.root_dir, "topics_index.json")
+        with open(output_path, "w") as f:
+            json.dump(topics, f, indent=2, ensure_ascii=False)
+        logger.info(f"Created topics index with {len(topics)} topics")
+
+    def generate_topics_md(self, topics: List[Dict]):
+        """Generate TOPICS.md documentation file in the root directory."""
+        # Group topics by category
+        categories_dict = defaultdict(list)
+        unique_topics = set()
+
+        for topic in topics:
+            topic_categories = topic.get("categories", [])
+            if isinstance(topic_categories, str):
+                topic_categories = [topic_categories]
+
+            # Add topic to each of its categories
+            for category in topic_categories:
+                # Create topic entry with aliases if they exist
+                topic_entry = topic["title"]
+                if "aliases" in topic and topic["aliases"]:
+                    aliases_str = ", ".join(topic["aliases"])
+                    topic_entry += f" (also covering: {aliases_str})"
+
+                categories_dict[category].append(topic_entry)
+                unique_topics.add(topic["title"])
+
+        # Sort categories and their topics
+        categories = sorted(categories_dict.keys())
+        for category in categories:
+            categories_dict[category].sort()
+
+        # Generate markdown content
+        content = []
+
+        # Add summary line
+        content.append(
+            f"*{len(categories)} categories for {len(unique_topics)} unique topics, with many appearing in multiple categories.*\n"
+        )
+
+        # Add table of contents as a single line
+        toc_items = []
+        for category in categories:
+            anchor = category.lower().replace(" ", "-")
+            toc_items.append(f"[{category}](#{anchor})")
+        content.append(" | ".join(toc_items))
+        content.append("")  # Empty line after ToC
+
+        # Add categories and their topics
+        for category in categories:
+            content.append(f"## {category}")
+            for topic in categories_dict[category]:
+                content.append(f"- {topic}")
+            content.append("")  # Empty line between categories
+
+        # Write to file in root directory
+        output_path = os.path.join(self.root_dir, "TOPICS.md")
+        with open(output_path, "w") as f:
+            f.write("\n".join(content))
+        logger.info("Created TOPICS.md documentation")
+
+    def build(self):
+        """Main function to build topics."""
+        try:
+            topics = self.build_topics()
+            self.write_topics_index(topics)
+            self.generate_topics_md(topics)
+        except Exception as e:
+            logger.error(f"Error during topics building: {str(e)}")
+            raise
+
+
+def main():
+    # Get the absolute path to the repository root (assuming script is in scripts/)
+    script_dir = Path(__file__).resolve().parent
+    root_dir = script_dir.parent
+
+    optech_topics_url = "https://bitcoinops.org/topics.json"
+    topics_dir = os.path.join(root_dir, "topics")
+
+    builder = TopicsBuilder(
+        optech_topics_url=optech_topics_url,
+        topics_dir=topics_dir,
+        root_dir=str(root_dir),
+    )
+    builder.build()
+
+
+if __name__ == "__main__":
+    main()

scripts/requirements.txt

@@ -0,0 +1,2 @@
+requests
+pyyaml