Recreate service containers with broken networks

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented here.
 
 Rascal follows semantic versioning. This has little consequence pre 1.0, so expect breaking changes.
 
+## 0.4.1 (2026-07-30)
+
+- Recreate a service container whose network no longer exists, instead of failing
+  to start it.
+
+
 ## 0.4.0 (2026-07-30)
 
 - Add `rascal run ENVIRONMENT -- COMMAND`, which runs a single command in an

features/shell.feature

@@ -86,6 +86,47 @@ Feature: Run "shell"
         """
 
 
+  Scenario: Recreate a service container whose network no longer exists
+    Given the following gitlab-ci config:
+      """
+      job:
+        image: job-image:latest
+        services:
+          - name: service-1-image:latest
+            alias: service-1
+      """
+      And the container "rascal-aruba-job_service-1" is attached to the network "a-network-that-is-gone"
+
+    When I successfully run `rascal shell job`
+    Then stdout should contain:
+        """
+        Removing container for aruba-job_service-1, its network no longer exists
+        """
+      And docker /container rm 91801.*/ should have been called
+      And docker /container create --name rascal-aruba-job_service-1/ should have been called
+      And stdout should contain:
+        """
+        Starting container for aruba-job_service-1
+        """
+
+
+  Scenario: Keep a service container that is attached to the current network
+    Given the following gitlab-ci config:
+      """
+      job:
+        image: job-image:latest
+        services:
+          - name: service-1-image:latest
+            alias: service-1
+      """
+      And the container "rascal-aruba-job_service-1" is attached to the network "deadbeef"
+
+    When I successfully run `rascal shell job`
+    Then stdout should not contain "its network no longer exists"
+      And docker /container rm/ should not have been called
+      And docker /container create --name rascal-aruba-job_service-1/ should not have been called
+
+
   Scenario: Run main container
     Given the following gitlab-ci config:
       """

features/step_definitions/docker_steps.rb

@@ -60,6 +60,9 @@ def output_for(*command)
       new_id = "%08x" % (@counter += 1)
       @containers[$1] = { id: new_id }
       new_id
+    when /container rm (\S*)/
+      @containers.delete_if { |_name, container| container[:id] == $1 }
+      $1
     when /container inspect (.*)/
       container = @containers.values.detect { |c| c[:id] == $1 }
       if container
@@ -121,6 +124,15 @@ def exit_status(success = true, code = nil)
   Rascal::Docker.interface.containers[name] = { id: Digest::SHA1.hexdigest("container-#{name}"), State: { Running: true } }
 end
 
+# The mock answers every `network ls` with "deadbeef", so that is the id of the
+# environment's current network. Any other id belongs to a network that is gone.
+Given("the container {string} is attached to the network {string}") do |name, network_id|
+  Rascal::Docker.interface.containers[name] = {
+    id: Digest::SHA1.hexdigest("container-#{name}"),
+    NetworkSettings: { Networks: { 'some-network' => { 'NetworkID' => network_id } } },
+  }
+end
+
 Given("the volume {string} exists") do |name|
   Rascal::Docker.interface.volumes[name] = { id: Digest::SHA1.hexdigest("volume-#{name}") }
 end

lib/rascal/docker/container.rb

@@ -23,23 +23,35 @@ def download_missing
       end
 
       def running?
-        if id
-          container_info = Docker.interface.run(
-            'container',
-            'inspect',
-            id,
-            output: :json,
-          ).first
-          !!container_info.dig('State', 'Running')
-        else
-          false
-        end
+        !!container_info&.dig('State', 'Running')
       end
 
       def exists?
         !!id
       end
 
+      # Docker resolves a container's network by the id it recorded when the
+      # container was created, and refuses to start it once that network is
+      # gone:
+      #
+      #   failed to set up container networking: network 36243f82141a... not found
+      #
+      # `docker network prune` (and `docker system prune`) leaves exactly that
+      # behind, because it removes networks whose containers are all stopped —
+      # the normal state of an environment between runs. Reconnecting does not
+      # help, since docker resolves the recorded id before it looks at the
+      # container's current endpoints, so the container has to go. Nothing in a
+      # service container is worth preserving; #start creates a new one.
+      def remove_if_network_missing(network)
+        return unless exists?
+        attached = attached_network_ids
+        return if attached.empty? || attached.include?(network.id)
+
+        say "Removing container for #{@name}, its network no longer exists"
+        remove_container
+        @id = nil
+      end
+
       def start(network: nil, network_alias: nil, volumes: [], env: {}, command: [])
         say "Starting container for #{@name}"
         create(network: network, network_alias: network_alias, volumes: volumes, env: env, command: command) unless exists?
@@ -123,6 +135,24 @@ def id
 
       private
 
+      # A fresh `docker container inspect`, or nil if there is no container.
+      def container_info
+        return unless id
+        Docker.interface.run(
+          'container',
+          'inspect',
+          id,
+          output: :json,
+        ).first
+      end
+
+      # Ids of the networks the container is attached to. Empty for a container
+      # that was created but never started.
+      def attached_network_ids
+        networks = container_info&.dig('NetworkSettings', 'Networks') || {}
+        networks.each_value.filter_map { |n| n['NetworkID'] }.reject(&:empty?).uniq
+      end
+
       def image_exists?
         Docker.interface.run(
           'image',

lib/rascal/docker/network.rb

@@ -39,11 +39,15 @@ def clean
         end
       end
 
+      # The full id, not the short one docker prints by default: a container
+      # records the network it is attached to by full id, and
+      # Container#remove_if_network_missing compares the two.
       def id
         @id ||= Docker.interface.run(
           'network',
           'ls',
           '--quiet',
+          '--no-trunc',
           '--filter', "name=^#{@prefixed_name}$",
           output: :id,
         )

lib/rascal/service.rb

@@ -17,6 +17,7 @@ def download_missing
 
     def start_if_stopped(network: nil)
       unless @container.running?
+        @container.remove_if_network_missing(network)
         if (container_id = @container.id)
           network.disconnect(container_id)
         end

lib/rascal/version.rb

@@ -1,3 +1,3 @@
 module Rascal
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end