Refactor Orchestrator with enhanced SSH handling, error management, and IP discovery support

- Implement retries for SSH-based job execution with configurable timeouts.
- Introduce `OrchestratorError` for consistent error handling across modules.
- Replace `virsh domifaddr` based guest IP discovery with a robust, multi-strategy approach.
- Refactor runner deployment and SSH-related utility functions for clarity.
- Add `thiserror` and `anyhow` dependencies for error management.
- Update persistence layer with improved error handling for database operations.

Signed-off-by: Till Wegmueller <toasterson@gmail.com>

Signed-off-by: Till Wegmueller <toasterson@gmail.com>

crates/orchestrator/src/error.rs

@@ -0,0 +1,84 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum OrchestratorError {
+    #[error("libvirt connect failed: {0}")]
+    LibvirtConnect(#[source] anyhow::Error),
+
+    #[error("define domain failed: {0}")]
+    LibvirtDefine(#[source] anyhow::Error),
+
+    #[error("lookup domain failed: {0}")]
+    LibvirtLookup(#[source] anyhow::Error),
+
+    #[error("domain operation failed: {0}")]
+    LibvirtDomain(#[source] anyhow::Error),
+
+    #[error("qemu-img not found or failed: {0}")]
+    QemuImg(#[source] anyhow::Error),
+
+    #[error("qemu-img convert failed: {0}")]
+    QemuImgConvert(#[source] anyhow::Error),
+
+    #[error("parse qemu-img info json failed: {0}")]
+    QemuImgParse(#[source] anyhow::Error),
+
+    #[error("mkisofs/genisoimage not found: {0}")]
+    MkIsoFs(#[source] anyhow::Error),
+
+    #[error("SSH session new failed")]
+    SshSessionNew,
+
+    #[error("tcp connect failed: {0}")]
+    TcpConnect(#[source] std::io::Error),
+
+    #[error("ssh handshake: {0}")]
+    SshHandshake(#[source] ssh2::Error),
+
+    #[error("ssh auth failed for {user}: {source}")]
+    SshAuth { user: String, #[source] source: ssh2::Error },
+
+    #[error("ssh not authenticated")]
+    SshNotAuthenticated,
+
+    #[error("sftp init failed: {0}")]
+    SftpInit(#[source] ssh2::Error),
+
+    #[error("open local runner: {0}")]
+    OpenLocalRunner(#[source] std::io::Error),
+
+    #[error("read runner: {0}")]
+    ReadRunner(#[source] std::io::Error),
+
+    #[error("sftp create: {0}")]
+    SftpCreate(#[source] ssh2::Error),
+
+    #[error("sftp write: {0}")]
+    SftpWrite(#[source] std::io::Error),
+
+    #[error("channel session: {0}")]
+    ChannelSession(#[source] ssh2::Error),
+
+    #[error("exec: {0}")]
+    Exec(#[source] ssh2::Error),
+
+    #[error("ssh keygen failed: {0}")]
+    SshKeygen(#[source] anyhow::Error),
+
+    #[error("encode pubkey: {0}")]
+    EncodePubKey(#[source] anyhow::Error),
+
+    #[error("encode privkey: {0}")]
+    EncodePrivKey(#[source] anyhow::Error),
+
+    #[error("failed to connect to database: {0}")]
+    DbConnect(#[source] anyhow::Error),
+
+    #[error("failed to apply migrations: {0}")]
+    DbMigrate(#[source] anyhow::Error),
+}
+
+// Helper conversions for common external error types into anyhow::Error where needed.
+impl From<virt::error::Error> for OrchestratorError {
+    fn from(e: virt::error::Error) -> Self { OrchestratorError::LibvirtDomain(anyhow::Error::new(e)) }
+}