Feat: Fix all deletion issues, add graceful shutdown to everything, new delete confirmation process

app/Actions/Application/StopApplication.php

@@ -3,50 +3,40 @@
 namespace App\Actions\Application;
 
 use App\Models\Application;
+use App\Actions\Server\CleanupDocker;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class StopApplication
 {
     use AsAction;
 
-    public function handle(Application $application, bool $previewDeployments = false)
+    public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true)
     {
-        if ($application->destination->server->isSwarm()) {
-            instant_remote_process(["docker stack rm {$application->uuid}"], $application->destination->server);
-
-            return;
-        }
-
-        $servers = collect([]);
-        $servers->push($application->destination->server);
-        $application->additional_servers->map(function ($server) use ($servers) {
-            $servers->push($server);
-        });
-        foreach ($servers as $server) {
-            if (! $server->isFunctional()) {
+        try {
+            $server = $application->destination->server;
+            if (!$server->isFunctional()) {
                 return 'Server is not functional';
             }
-            if ($previewDeployments) {
-                $containers = getCurrentApplicationContainerStatus($server, $application->id, includePullrequests: true);
-            } else {
-                $containers = getCurrentApplicationContainerStatus($server, $application->id, 0);
-            }
-            if ($containers->count() > 0) {
-                foreach ($containers as $container) {
-                    $containerName = data_get($container, 'Names');
-                    if ($containerName) {
-                        instant_remote_process(command: ["docker stop --time=30 $containerName"], server: $server, throwError: false);
-                        instant_remote_process(command: ["docker rm $containerName"], server: $server, throwError: false);
-                        instant_remote_process(command: ["docker rm -f {$containerName}"], server: $server, throwError: false);
-                    }
-                }
+            ray('Stopping application: ' . $application->name);
+
+            if ($server->isSwarm()) {
+                instant_remote_process(["docker stack rm {$application->uuid}"], $server);
+                return;
             }
+
+            $containersToStop = $application->getContainersToStop($previewDeployments);
+            $application->stopContainers($containersToStop, $server);
+
             if ($application->build_pack === 'dockercompose') {
-                // remove network
-                $uuid = $application->uuid;
-                instant_remote_process(["docker network disconnect {$uuid} coolify-proxy"], $server, false);
-                instant_remote_process(["docker network rm {$uuid}"], $server, false);
+                $application->delete_connected_networks($application->uuid);
+            }
+
+            if ($dockerCleanup) {
+                CleanupDocker::run($server, true);
             }
+        } catch (\Exception $e) {
+            ray($e->getMessage());
+            return $e->getMessage();
         }
     }
 }

app/Actions/Database/StopDatabase.php

@@ -10,25 +10,67 @@
 use App\Models\StandaloneMysql;
 use App\Models\StandalonePostgresql;
 use App\Models\StandaloneRedis;
+use Illuminate\Support\Facades\Process;
 use Lorisleiva\Actions\Concerns\AsAction;
+use App\Actions\Server\CleanupDocker;
 
 class StopDatabase
 {
     use AsAction;
 
-    public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database)
+    public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, bool $isDeleteOperation = false, bool $dockerCleanup = true)
     {
         $server = $database->destination->server;
-        if (! $server->isFunctional()) {
+        if (!$server->isFunctional()) {
             return 'Server is not functional';
         }
 
-        instant_remote_process(command: ["docker stop --time=30 $database->uuid"], server: $server, throwError: false);
-        instant_remote_process(command: ["docker rm $database->uuid"], server: $server, throwError: false);
-        instant_remote_process(command: ["docker rm -f $database->uuid"], server: $server, throwError: false);
+        $this->stopContainer($database, $database->uuid, 300);
+        if (!$isDeleteOperation) {
+            $this->deleteConnectedNetworks($database->uuid, $server); //Probably not needed as DBs do not have a network normally
+            if ($dockerCleanup) {
+                CleanupDocker::run($server, true);
+            }
+        }
 
         if ($database->is_public) {
             StopDatabaseProxy::run($database);
         }
+
+        return 'Database stopped successfully';
+    }
+
+    private function stopContainer($database, string $containerName, int $timeout = 300): void
+    {
+        $server = $database->destination->server;
+
+        $process = Process::timeout($timeout)->start("docker stop --time=$timeout $containerName");
+
+        $startTime = time();
+        while ($process->running()) {
+            if (time() - $startTime >= $timeout) {
+                $this->forceStopContainer($containerName, $server);
+                break;
+            }
+            usleep(100000);
+        }
+
+        $this->removeContainer($containerName, $server);
+    }
+
+    private function forceStopContainer(string $containerName, $server): void
+    {
+        instant_remote_process(command: ["docker kill $containerName"], server: $server, throwError: false);
+    }
+
+    private function removeContainer(string $containerName, $server): void
+    {
+        instant_remote_process(command: ["docker rm -f $containerName"], server: $server, throwError: false);
+    }
+
+    private function deleteConnectedNetworks($uuid, $server)
+    {
+        instant_remote_process(["docker network disconnect {$uuid} coolify-proxy"], $server, false);
+        instant_remote_process(["docker network rm {$uuid}"], $server, false);
     }
 }

app/Actions/Service/DeleteService.php

@@ -3,17 +3,18 @@
 namespace App\Actions\Service;
 
 use App\Models\Service;
+use App\Actions\Server\CleanupDocker;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class DeleteService
 {
     use AsAction;
 
-    public function handle(Service $service)
+    public function handle(Service $service, bool $deleteConfigurations, bool $deleteVolumes, bool $dockerCleanup, bool $deleteConnectedNetworks)
     {
         try {
             $server = data_get($service, 'server');
-            if ($server->isFunctional()) {
+            if ($deleteVolumes && $server->isFunctional()) {
                 $storagesToDelete = collect([]);
 
                 $service->environment_variables()->delete();
@@ -33,13 +34,29 @@ public function handle(Service $service)
                 foreach ($storagesToDelete as $storage) {
                     $commands[] = "docker volume rm -f $storage->name";
                 }
-                $commands[] = "docker rm -f $service->uuid";
 
-                instant_remote_process($commands, $server, false);
+                // Execute volume deletion first, this must be done first otherwise volumes will not be deleted.
+                if (!empty($commands)) {
+                    foreach ($commands as $command) {
+                        $result = instant_remote_process([$command], $server, false);
+                        if ($result !== 0) {
+                            ray("Failed to execute: $command");
+                        }
+                    }
+                }
+            }
+
+            if ($deleteConnectedNetworks) {
+                $service->delete_connected_networks($service->uuid);
             }
+
+            instant_remote_process(["docker rm -f $service->uuid"], $server, throwError: false);
         } catch (\Exception $e) {
             throw new \Exception($e->getMessage());
         } finally {
+            if ($deleteConfigurations) {
+                $service->delete_configurations();
+            }
             foreach ($service->applications()->get() as $application) {
                 $application->forceDelete();
             }
@@ -50,6 +67,11 @@ public function handle(Service $service)
                 $task->delete();
             }
             $service->tags()->detach();
+            $service->forceDelete();
+
+            if ($dockerCleanup) {
+                CleanupDocker::run($server, true);
+            }
         }
     }
 }

app/Actions/Service/StopService.php

@@ -3,46 +3,33 @@
 namespace App\Actions\Service;
 
 use App\Models\Service;
+use App\Actions\Server\CleanupDocker;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class StopService
 {
     use AsAction;
 
-    public function handle(Service $service)
+    public function handle(Service $service, bool $isDeleteOperation = false, bool $dockerCleanup = true)
     {
         try {
             $server = $service->destination->server;
-            if (! $server->isFunctional()) {
+            if (!$server->isFunctional()) {
                 return 'Server is not functional';
             }
-            ray('Stopping service: '.$service->name);
-            $applications = $service->applications()->get();
-            foreach ($applications as $application) {
-                if ($applications->count() < 6) {
-                    instant_remote_process(command: ["docker stop --time=10 {$application->name}-{$service->uuid}"], server: $server, throwError: false);
-                }
-                instant_remote_process(command: ["docker rm {$application->name}-{$service->uuid}"], server: $server, throwError: false);
-                instant_remote_process(command: ["docker rm -f {$application->name}-{$service->uuid}"], server: $server, throwError: false);
-                $application->update(['status' => 'exited']);
-            }
-            $dbs = $service->databases()->get();
-            foreach ($dbs as $db) {
-                if ($dbs->count() < 6) {
 
-                    instant_remote_process(command: ["docker stop --time=10 {$db->name}-{$service->uuid}"], server: $server, throwError: false);
+            $containersToStop = $service->getContainersToStop();
+            $service->stopContainers($containersToStop, $server);
+
+            if (!$isDeleteOperation) {
+                $service->delete_connected_networks($service->uuid);
+                if ($dockerCleanup) {
+                    CleanupDocker::run($server, true);
                 }
-                instant_remote_process(command: ["docker rm {$db->name}-{$service->uuid}"], server: $server, throwError: false);
-                instant_remote_process(command: ["docker rm -f {$db->name}-{$service->uuid}"], server: $server, throwError: false);
-                $db->update(['status' => 'exited']);
             }
-            instant_remote_process(["docker network disconnect {$service->uuid} coolify-proxy"], $service->server);
-            instant_remote_process(["docker network rm {$service->uuid}"], $service->server);
         } catch (\Exception $e) {
             ray($e->getMessage());
-
             return $e->getMessage();
         }
-
     }
 }

app/Jobs/ApplicationDeploymentJob.php

@@ -29,6 +29,7 @@
 use Illuminate\Support\Collection;
 use Illuminate\Support\Sleep;
 use Illuminate\Support\Str;
+use Illuminate\Support\Facades\Process;
 use RuntimeException;
 use Symfony\Component\Yaml\Yaml;
 use Throwable;
@@ -2211,20 +2212,40 @@ private function build_image()
         $this->application_deployment_queue->addLogEntry('Building docker image completed.');
     }
 
-    /**
-     * @param  int  $timeout  in seconds
-     */
-    private function graceful_shutdown_container(string $containerName, int $timeout = 30)
+    private function graceful_shutdown_container(string $containerName, int $timeout = 300)
     {
         try {
-            $this->execute_remote_command(
-                ["docker stop --time=$timeout $containerName", 'hidden' => true, 'ignore_errors' => true],
-                ["docker rm $containerName", 'hidden' => true, 'ignore_errors' => true]
-            );
+            $process = Process::timeout($timeout)->start("docker stop --time=$timeout $containerName");
+
+            $startTime = time();
+            while ($process->running()) {
+                if (time() - $startTime >= $timeout) {
+                    $this->execute_remote_command(
+                        ["docker kill $containerName", 'hidden' => true, 'ignore_errors' => true]
+                    );
+                    break;
+                }
+                usleep(100000);
+            }
+
+            $isRunning = $this->execute_remote_command(
+                ["docker inspect -f '{{.State.Running}}' $containerName", 'hidden' => true, 'ignore_errors' => true]
+            ) === 'true';
+
+            if ($isRunning) {
+                $this->execute_remote_command(
+                    ["docker kill $containerName", 'hidden' => true, 'ignore_errors' => true]
+                );
+            }
         } catch (\Exception $error) {
-            // report error if needed
+            $this->application_deployment_queue->addLogEntry("Error stopping container $containerName: " . $error->getMessage(), 'stderr');
         }
 
+        $this->remove_container($containerName);
+    }
+
+    private function remove_container(string $containerName)
+    {
         $this->execute_remote_command(
             ["docker rm -f $containerName", 'hidden' => true, 'ignore_errors' => true]
         );
@@ -2240,7 +2261,7 @@ private function stop_running_container(bool $force = false)
                 $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id);
                 if ($this->pull_request_id === 0) {
                     $containers = $containers->filter(function ($container) {
-                        return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== $this->container_name.'-pr-'.$this->pull_request_id;
+                        return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== $this->container_name . '-pr-' . $this->pull_request_id;
                     });
                 }
                 $containers->each(function ($container) {