Merge pull request #68 from efabless/requirements.txt

Requirements.txt

.github/workflows/caravel_cocotb.yml

@@ -35,6 +35,10 @@ jobs:
     - name: Run Flake8
       run: flake8 $GITHUB_WORKSPACE/cocotb --config $GITHUB_WORKSPACE/cocotb/.flake8
 
+    - name: download docker
+      run: |
+        docker pull efabless/dv:cocotb
+
     - name: Install caravel_cocotb
       run: |
         cd $GITHUB_WORKSPACE/cocotb
@@ -43,4 +47,4 @@ jobs:
     - name: Run main.py
       run: |
         cd $GITHUB_WORKSPACE/cocotb/caravel_cocotb/CI
-        python3 main.py
+        python3 main.py

cocotb/README.md

@@ -276,6 +276,10 @@ New directory named ``sim`` would be created under ``<repo root>/cocotb/`` or to
 # Update design_info.yaml 
 <!-- start Update design_info include -->
 
+> **Note**: This step is required only if make setup isn't used after cloning <caravel_user_project> 
+
+> After any change `make setup-cocotb` can be used.
+
 ``design_info.yaml`` are used to reference all the needed repos and paths needed to run the tests:
 
 fill  it like the following
@@ -311,6 +315,42 @@ fill  it like the following
   # optional email address to send the results to 
   emailto: [None]
 ```
-> **Note**: This step is required only in the first run.
 
 <!-- end Update design_info include -->
+
+# HDL include files format 
+<!-- start Update include files format include -->
+Include files from ``<caravel_user_project>/verilog/include`` like  ``includes.rtl.caravel_user_project``, ``includes.gl.caravel_user_project`` and ``includes.sdf.caravel_user_project`` are used to reference all the needed repos and paths needed to run the tests:
+
+The legacy format to reference files is:
+- support verilog file include
+   ```bash
+   -v $(USER_PROJECT_VERILOG)/rtl/user_project_wrapper.v
+   ```
+
+Coctb flow supports this format in addition to other formats:
+- support systemVerilg file include
+   ```bash
+   -sv  $(USER_PROJECT_VERILOG)/rtl/counter.sv
+   ```
+- support wild card use
+   ```bash
+   -sv $(USER_PROJECT_VERILOG)/rtl/peripherals/*.sv
+   ```
+- support add search path for `.vh` and `.svh` files
+   ```bash
+   -I $(USER_PROJECT_VERILOG)/rtl/peripherals
+   ```
+
+<!-- end Update include files format include -->
+
+# Unexisted python modules 
+<!-- start unexisted python modules include -->
+If the testbench use python modules that are not installed in the docker image, there are 2 options: 
+
+1. use `-no_docker` option to run without docker if you have all the required tools installed.
+2. Add requirments file to `<caravel_user_project>/verilog/dv/cocotb/requirements.txt` 
+   ```txt
+   rich==12.0.1
+   ```
+<!-- end unexisted python modules include -->

cocotb/caravel_cocotb/CI/gen_run_command.py

@@ -78,7 +78,7 @@ def all_switches_chooser(self):
         self.test_list_chooser = RandomChooser([f"{self.cocotb_path}/test_io10/test_io10.yaml", f"{self.cocotb_path}/test_mgmt_gpio/test_mgmt_gpio.yaml"])
         self.design_infos_chooser = RandomChooser([None, f"{self.cocotb_path}/design_info.yaml"])
         self.sims_chooser = RandomChooser([None, 'RTL', "GL", "GL_SDF", "RTL GL", "GL RTL"])
-        self.tags_chooser = RandomChooser([''.join(random.choice(string.ascii_lowercase) for _ in range(i)) for i in range(5, 10)])  # change string length to make sure tag is unique
+        self.tags_chooser = RandomChooser([''.join(random.choice(string.ascii_lowercase) for _ in range(i)) for i in range(5, 40)])  # change string length to make sure tag is unique
         self.max_errors_chooser = RandomChooser([None, "7", "5", "100"])
         self.corners_chooser = RandomChooser([None, "nom-t", "nom-f", "nom-s", "max-t", "max-f", "max-s", "min-t", "min-f", "min-s"])
         self.seed_chooser = RandomChooser([None] + [random.randint(0, 100000) for _ in range(5)])
@@ -87,7 +87,7 @@ def all_switches_chooser(self):
         self.macros_chooser = RandomChooser([None, "USE_MACRO_1", "USE_MACRO_2"])
         self.sim_paths_chooser = RandomChooser([None, os.path.abspath(os.path.join(self.cocotb_path, ".."))])
         # self.verbosities_chooser = RandomChooser([None, "quiet", "normal", "debug"])
-        self.verbosities_chooser = RandomChooser(["quiet"])  # to speed sims
+        self.verbosities_chooser = RandomChooser(["debug"])  # to speed sims
         self.compiles_chooser = RandomChooser([None, True])
         self.check_commits_chooser = RandomChooser([None, True])
         self.run_location = RandomChooser([self.cocotb_path, os.path.abspath(os.path.join(self.cocotb_path, "..", ".."))])

cocotb/caravel_cocotb/CI/setup_env.py

@@ -9,7 +9,6 @@ def __init__(self, paths):
         super().__init__()
         self.paths = paths
         self.clone_needed_repos()
-        self.pull_cocotb_docker()
 
     def clone_needed_repos(self):
         self.clone_repo(repo_url="https://github.com/efabless/caravel.git", target_dir=self.paths.caravel_root, depth=1, branch="main")

cocotb/caravel_cocotb/docker/DockerFile

@@ -44,21 +44,6 @@ RUN apt-get update && \
     libglib2.0-dev\
     gperf
 
-# Upgrade pip
-RUN pip3 install --upgrade pip
-
-# Install Python packages using pip
-RUN pip3 install --upgrade\
-    cocotb \
-    cocotb_coverage \
-    cocotb-bus \
-    coverage \
-    loremipsum \
-    oyaml \
-    prettytable \
-    anytree\
-    caravel-cocotb
-
 # Install Verilator from source v5.012
 RUN git clone https://github.com/verilator/verilator && \
     cd verilator && \
@@ -100,9 +85,45 @@ ENV PATH="/opt/riscv/bin:${PATH}"
 RUN apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Create a non-root user
+RUN useradd -ms /bin/bash myuser
+
 # Set the working directory
 WORKDIR /app
 
+# Copy the application code
+COPY . /app
+
+# Change ownership to the non-root user
+RUN chown -R myuser:myuser /app
+
+# Switch to the non-root user
+USER myuser
+
+
+# Upgrade pip
+RUN pip3 install --upgrade pip
+
+# Install Python packages using pip
+RUN pip3 install --upgrade\
+    cocotb \
+    cocotb_coverage \
+    cocotb-bus \
+    coverage \
+    loremipsum \
+    oyaml \
+    prettytable \
+    anytree\
+    caravel-cocotb
+
+
+USER root
+ENV PATH /home/myuser/.local/bin:$PATH
+
+# Add any additional setup or changes here
+
+USER myuser
+
 # Default command when the container starts
 CMD ["bash"]
 

cocotb/caravel_cocotb/docs/source/usage.rst

@@ -82,3 +82,19 @@ Update design_info.yaml
    :parser: markdown
    :start-after: <!-- start Update design_info include -->
    :end-before: <!-- end Update design_info include -->
+
+HDL include files format 
+*************************
+
+.. include:: ../../../README.md
+   :parser: markdown
+   :start-after: <!-- start Update include files format include -->
+   :end-before: <!-- end Update include files format include -->
+
+Unexisted python modules 
+*************************
+
+.. include:: ../../../README.md
+   :parser: markdown
+   :start-after: <!-- start unexisted python modules include -->
+   :end-before: <!-- end unexisted python modules include -->

cocotb/caravel_cocotb/scripts/merge_coverage.py

@@ -69,8 +69,8 @@ def merge_fun_cov(path, reports_path=None):
         tree = {}
         for key in yaml_file_object:
             level = key.count(".")
-            nodes = key.split(".")
-            key_parent = key.replace(f".{nodes[level]}", "")
+            last_node_index = key.rfind(".")
+            key_parent = key[:last_node_index] if last_node_index != -1 else key
             if level == 0:
                 root = key
                 tree[key] = Node(key=key)

cocotb/caravel_cocotb/scripts/verify_cocotb/DockerProcess.py

@@ -0,0 +1,57 @@
+import subprocess
+import os
+
+class DockerProcess:
+    def __init__(self, image_name, user_project_path=None, sim_path=None) -> None:
+        self.image_name = image_name
+        self.user_project_path = user_project_path
+        self.sim_path = sim_path
+
+    def run(self):
+        # pull/update docker image
+        self.pull_docker_image()
+        # update docker image with pip commands if requirements.txt exists
+        if os.path.exists(f"{self.user_project_path}/verilog/dv/cocotb/requirements.txt"):
+            self.write_docker_file()
+            self.build_docker_image()
+
+    def pull_docker_image(self):
+        # Check if the image exists locally
+        try:
+            subprocess.run(["docker", "inspect", self.image_name], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            print(f"check update for docker image {self.image_name}.")
+            command = ["docker", "pull", "-q", f"{self.image_name}"]
+        except subprocess.CalledProcessError:
+            print(f"pulling  docker image {self.image_name}.")
+            command = ["docker", "pull", f"{self.image_name}"]
+        try:
+            # Run the docker pull command
+            subprocess.run(command, check=True)
+        except subprocess.CalledProcessError as e:
+            print(f"Error: Failed to pull {self.image_name}")
+            print(e)
+
+    def build_docker_image(self):
+        try:
+            # Build the Docker image using subprocess
+            subprocess.run(["docker", "build", "-t", self.image_name, "-f", f"{self.sim_path}/Dockerfile", "."], check=True)
+            print(f"Docker image '{self.image_name}' built successfully.")
+        except subprocess.CalledProcessError as e:
+            print(f"Error building Docker image: {e}")
+
+    def write_docker_file(self):
+        with open(f"{self.sim_path}/Dockerfile", "w") as f:
+            with open("requirements.txt", "r") as file:
+                requirements = file.readlines()
+            requirements = [requirement.strip() for requirement in requirements]
+
+            f.write("# Use the efabless/dv:cocotb base image\n")
+            f.write("FROM efabless/dv:cocotb\n")
+            f.write("\n")
+            # f.write("# Copy requirements.txt into the container\n")
+            # f.write("WORKDIR /app\n")
+            # f.write(f"COPY {self.user_project_path}/verilog/dv/cocotb/requirements.txt .\n")
+            f.write("\n")
+            f.write("# Install additional packages\n")
+            f.write(f"RUN pip install --upgrade {' '.join(requirements)}")
+            f.write("\n")

cocotb/caravel_cocotb/scripts/verify_cocotb/RunFlow.py

@@ -8,8 +8,7 @@
 import re
 import logging
 import random
-import subprocess
-
+from caravel_cocotb.scripts.verify_cocotb.DockerProcess import DockerProcess
 
 def check_valid_mail_addr(address):
     pat = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b"
@@ -32,7 +31,7 @@ def __init__(self, args) -> None:
         self.set_tag()
         self.set_args(design_info)
         self.set_config_script(design_info)
-        self.pull_docker_image("efabless/dv:cocotb")
+        DockerProcess("efabless/dv:cocotb", self.paths.USER_PROJECT_ROOT, f"{self.paths.SIM_PATH}/{self.args.tag}").run()
         RunRegression(self.args, self.paths, self.logger)
 
     def configure_logger(self):
@@ -214,21 +213,6 @@ def get_design_info(self):
         design_info = yaml.safe_load(yaml_file)
         return design_info
 
-    def pull_docker_image(self, image_full_name):
-        # Check if the image exists locally
-        try:
-            subprocess.run(["docker", "inspect", image_full_name], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            print(f"check update for docker image {image_full_name}.")
-        except subprocess.CalledProcessError:
-            print(f"pulling  docker image {image_full_name}.")
-        command = ["docker", "pull", "-q", f"{image_full_name}"]
-        try:
-            # Run the docker pull command
-            subprocess.run(command, check=True)
-        except subprocess.CalledProcessError as e:
-            print(f"Error: Failed to pull {image_full_name}")
-            print(e)
-
 
 class CocotbArgs:
     def __init__(