feat: install Python and Node packages in container (#1220)

Python, Node, and Ruby packages were all installed in the local
workspace at runtime. This slows down initial runs and creates the
potential for people to use different versions of packages (the Node
lock file was not actually being used). It also creates permissions
issues when switching between different container runtimes (I've been
switching between Podman and the Devcontainer lately).

This installs the Python and Node packages in the container and updates
the `npm` installation process to actually use `package-lock.json`.

~~I haven't tested with Singularity~~ (I'm a Podman/Docker user), but
everything is passing with Podman locally.

I'll follow up with another PR later to migrate some of the Ruby
packages to the container as well, but that one is trickier because some
of the Ruby packages are local to the UDB project.

Author: jordancarlin

.devcontainer/Dockerfile

@@ -42,4 +42,19 @@ RUN apt-get clean autoclean
 RUN apt-get autoremove -y
 RUN rm -rf /var/lib/{apt,dpkg,cache,log}/*
 
+# Create Python virtual environment and install packages
+COPY requirements.txt /tmp/requirements.txt
+RUN /usr/bin/python3 -m venv /opt/venv && \
+    /opt/venv/bin/pip install -r /tmp/requirements.txt && \
+    rm /tmp/requirements.txt
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Install npm packages globally in the container
+COPY package.json package-lock.json /opt/node/
+RUN cd /opt/node && \
+    /usr/bin/npm ci && \
+    rm /opt/node/package.json /opt/node/package-lock.json
+ENV NODE_PATH="/opt/node/node_modules"
+ENV PATH="/opt/node/node_modules/.bin:$PATH"
+
 WORKDIR /workspace

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "RISC-V UnifiedDB Dev",
-  "build": { "dockerfile": "Dockerfile" },
+  "build": { "dockerfile": "Dockerfile", "context": ".." },
   "updateContentCommand": "${containerWorkspaceFolder}/.devcontainer/updateContentCommand.sh",
   "onCreateCommand": "${containerWorkspaceFolder}/.devcontainer/onCreateCommand.sh",
   "remoteEnv": {

.devcontainer/onCreateCommand.sh

@@ -1,4 +1,3 @@
 #!/bin/bash
 
-npm i
 bundle install

Rakefile

@@ -119,7 +119,7 @@ namespace :test do
   # "Run the cross-validation against LLVM"
   task :llvm do
       begin
-        sh "#{$root}/.home/.venv/bin/python3 -m pytest ext/auto-inst/test_parsing.py -v"
+        sh "/opt/venv/bin/python3 -m pytest ext/auto-inst/test_parsing.py -v"
       rescue => e
         raise unless e.message.include?("status (5)") # don't fail on skipped tests
     end

backends/cfg_html_doc/html_gen.rake

@@ -119,9 +119,9 @@ rule %r{#{$root}/gen/cfg_html_doc/.*/antora/playbook.yaml} => proc { |tname|
         snapshot: true
       supplemental_files:
       - path: css/vendor/tabs.css
-        contents: #{$root}/node_modules/@asciidoctor/tabs/dist/css/tabs.css
+        contents: /opt/node/node_modules/@asciidoctor/tabs/dist/css/tabs.css
       - path: js/vendor/tabs.js
-        contents: #{$root}/node_modules/@asciidoctor/tabs/dist/js/tabs.js
+        contents: /opt/node/node_modules/@asciidoctor/tabs/dist/js/tabs.js
       - path: partials/footer-scripts.hbs
         contents: |
           <script id="site-script" src="{{{uiRootPath}}}/js/site.js" data-ui-root-path="{{{uiRootPath}}}"></script>
@@ -234,10 +234,9 @@ rule %r{#{$root}/\.stamps/html-gen-.*\.stamp} => proc { |tname|
   playbook_path = $root / "gen" / "cfg_html_doc" / config_name / "antora" / "playbook.yaml"
   Rake::Task[playbook_path].invoke
 
-  $logger.info "Running Antora under npm to create #{config_name}"
-
+  $logger.info "Running Antora to create #{config_name}"
   sh [
-    "npm exec -- antora",
+    "/opt/node/node_modules/.bin/antora",
     "--stacktrace",
     "generate",
     "--cache-dir=#{$root}/.home/.antora",
@@ -247,7 +246,7 @@ rule %r{#{$root}/\.stamps/html-gen-.*\.stamp} => proc { |tname|
     playbook_path
   ].join(" ")
 
-  $logger.info "Done running Antora under npm to create #{config_name}"
+  $logger.info "Done running Antora to create #{config_name}"
 
   FileUtils.touch t.name
 end

backends/ext_pdf_doc/tasks.rake

@@ -34,7 +34,7 @@ rule %r{#{$resolver.gen_path}/ext_pdf_doc/.*/pdf/.*_extension\.pdf} => proc { |t
     "-a pdf-theme=#{ENV['THEME']}",
     "-a pdf-fontsdir=#{$root}/ext/docs-resources/fonts",
     "-a imagesdir=#{$root}/ext/docs-resources/images",
-    "-a wavedrom=#{$root}/node_modules/.bin/wavedrom-cli",
+    "-a wavedrom=/opt/node/node_modules/.bin/wavedrom-cli",
     "-r asciidoctor-diagram",
     "-r idl_highlighter",
     "-o #{t.name}",
@@ -63,7 +63,7 @@ rule %r{#{$resolver.gen_path}/ext_pdf_doc/.*/html/.*_extension\.html} => proc {
     "-v",
     "-a toc",
     "-r asciidoctor-diagram",
-    "-a wavedrom=#{$root}/node_modules/.bin/wavedrom-cli",
+    "-a wavedrom=/opt/node/node_modules/.bin/wavedrom-cli",
     "-o #{t.name}",
     adoc_file
   ].join(" ")

backends/generators/tasks.rake

@@ -28,9 +28,9 @@ namespace :gen do
     inst_dir = cfg_arch.path / "inst"
     csr_dir = cfg_arch.path / "csr"
 
-    # Run the Go generator script using the same Python environment
+    # Run the Go generator script
     # Note: The script uses --output not --output-dir
-    sh "#{$root}/.home/.venv/bin/python3 #{$root}/backends/generators/Go/go_generator.py --inst-dir=#{inst_dir} --csr-dir=#{csr_dir} --output=#{output_dir}inst.go"
+    sh "/opt/venv/bin/python3 #{$root}/backends/generators/Go/go_generator.py --inst-dir=#{inst_dir} --csr-dir=#{csr_dir} --output=#{output_dir}inst.go"
   end
 
   desc <<~DESC
@@ -79,9 +79,9 @@ namespace :gen do
     resolved_codes_file.flush
 
     begin
-      # Run the C header generator script using the same Python environment
+      # Run the C header generator script
       # The script generates encoding.h for inclusion in C programs
-      sh "#{$root}/.home/.venv/bin/python3 #{$root}/backends/generators/c_header/generate_encoding.py --inst-dir=#{inst_dir} --csr-dir=#{csr_dir} --ext-dir=#{ext_dir} --resolved-codes=#{resolved_codes_file.path} --output=#{output_dir}encoding.out.h --include-all"
+      sh "/opt/venv/bin/python3 #{$root}/backends/generators/c_header/generate_encoding.py --inst-dir=#{inst_dir} --csr-dir=#{csr_dir} --ext-dir=#{ext_dir} --resolved-codes=#{resolved_codes_file.path} --output=#{output_dir}encoding.out.h --include-all"
     ensure
       resolved_codes_file.close
       resolved_codes_file.unlink

backends/instructions_appendix/all_instructions.golden.adoc

@@ -1,6 +1,6 @@
 = Instruction Appendix
 :doctype: book
-:wavedrom: /workspaces/riscv-unified-db/node_modules/.bin/wavedrom-cli
+:wavedrom: /opt/node/node_modules/.bin/wavedrom-cli
 // Now the document header is complete and the wavedrom attribute is active.
 
 

backends/instructions_appendix/templates/instructions.adoc.erb

@@ -1,6 +1,6 @@
 = Instruction Appendix
 :doctype: book
-:wavedrom: <%= $root %>/node_modules/.bin/wavedrom-cli
+:wavedrom: /opt/node/node_modules/.bin/wavedrom-cli
 // Now the document header is complete and the wavedrom attribute is active.
 
 <% instructions.sort_by(&:name).each do |inst| %>

backends/manual/tasks.rake

@@ -434,10 +434,10 @@ namespace :gen do
     playbook_path = MANUAL_GEN_DIR / ENV["MANUAL_NAME"] / "top" / output_hash / "antora" / "playbook" / "playbook.yml"
     Rake::Task[playbook_path].invoke
 
-    $logger.info "Running Antora under npm for HTML site to create '#{MANUAL_GEN_DIR / ENV['MANUAL_NAME'] / output_hash / 'html'}'"
+    $logger.info "Running Antora for HTML site to create '#{MANUAL_GEN_DIR / ENV['MANUAL_NAME'] / output_hash / 'html'}'"
 
     sh [
-      "npm exec -- antora",
+      "/opt/node/node_modules/.bin/antora",
       "--stacktrace",
       "generate",
       "--cache-dir=#{$root}/.home/.antora",
@@ -447,7 +447,7 @@ namespace :gen do
       playbook_path.to_s
     ].join(" ")
 
-    $logger.info "Done running Antora under npm for HTML site to create '#{MANUAL_GEN_DIR / ENV['MANUAL_NAME'] / output_hash / 'html'}'"
+    $logger.info "Done running Antora for HTML site to create '#{MANUAL_GEN_DIR / ENV['MANUAL_NAME'] / output_hash / 'html'}'"
   end
 end