CI (Work in progress): Use nix for building in CI

.woodpecker/agent-release.yml

@@ -0,0 +1,37 @@
+depends_on:
+  - build
+  - test
+  - lint
+
+when:
+  - event: push
+    branch:
+      - ${CI_REPO_DEFAULT_BRANCH}
+
+matrix:
+  BUILD_TARGET:
+    - x86_64
+    - aarch64
+
+steps:
+  build:
+    image: nixos/nix:latest
+    commands:
+      - nix --extra-experimental-features "nix-command flakes" build ".#agent.${BUILD_TARGET}"
+      - mkdir -p "dist/${BUILD_TARGET}-unknown-linux-musl"
+      - cp result/bin/prymn_agent "dist/${BUILD_TARGET}-unknown-linux-musl/"
+
+  release:
+    image: woodpeckerci/plugin-s3
+    settings:
+      bucket: prymn-static
+      region: auto
+      endpoint:
+        from_secret: R2_STATIC_ADDRESS
+      access_key:
+        from_secret: R2_DEPLOY_ACCESS_KEY
+      secret_key:
+        from_secret: R2_DEPLOY_SECRET_KEY
+      source: dist/**/*
+      target: /agent
+      strip_prefix: dist/

.woodpecker/build.yml

@@ -5,7 +5,7 @@ when:
       - ${CI_REPO_DEFAULT_BRANCH}
 
 variables:
-  - &rust_image 'git.nikos.gg/prymn/images/rust:1794cee2'
+  - &rust_image 'git.nikos.gg/prymn/images/rust:33331dfc'
   - &elixir_image 'git.nikos.gg/prymn/images/elixir:1794cee2'
   - &cache_settings
       endpoint: https://75178f9eca227dea51b3db4db2c15a5a.r2.cloudflarestorage.com

.woodpecker/get-prymn-release.yml

@@ -0,0 +1,19 @@
+when:
+  - event: push
+    branch:
+      - ${CI_REPO_DEFAULT_BRANCH}
+
+steps:
+  release:
+    image: woodpeckerci/plugin-s3
+    settings:
+      bucket: prymn-static
+      region: auto
+      endpoint:
+        from_secret: R2_STATIC_ADDRESS
+      access_key:
+        from_secret: R2_DEPLOY_ACCESS_KEY
+      secret_key:
+        from_secret: R2_DEPLOY_SECRET_KEY
+      source: get_prymn.sh
+      target: /agent

.woodpecker/lint.yml

@@ -9,7 +9,7 @@ when:
       - ${CI_REPO_DEFAULT_BRANCH}
 
 variables:
-  - &rust_image 'git.nikos.gg/prymn/images/rust:1794cee2'
+  - &rust_image 'git.nikos.gg/prymn/images/rust:33331dfc'
   - &elixir_image 'git.nikos.gg/prymn/images/elixir:1794cee2'
   - &when_elixir
     - path:

.woodpecker/test.yml

@@ -15,7 +15,7 @@ services:
       - POSTGRES_PASSWORD=postgres
 
 variables:
-  - &rust_image 'git.nikos.gg/prymn/images/rust:1794cee2'
+  - &rust_image 'git.nikos.gg/prymn/images/rust:33331dfc'
   - &elixir_image 'git.nikos.gg/prymn/images/elixir:1794cee2'
   - &cache_settings
       endpoint: https://75178f9eca227dea51b3db4db2c15a5a.r2.cloudflarestorage.com

Cargo.lock

@@ -95,28 +95,6 @@ version = "1.0.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
 
-[[package]]
-name = "async-stream"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
-dependencies = [
- "async-stream-impl",
- "futures-core",
- "pin-project-lite",
-]
-
-[[package]]
-name = "async-stream-impl"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.31",
-]
-
 [[package]]
 name = "async-trait"
 version = "0.1.73"
@@ -391,17 +369,6 @@ version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
 
-[[package]]
-name = "futures-macro"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.31",
-]
-
 [[package]]
 name = "futures-sink"
 version = "0.3.28"
@@ -422,7 +389,6 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
 dependencies = [
  "futures-core",
  "futures-io",
- "futures-macro",
  "futures-task",
  "memchr",
  "pin-project-lite",
@@ -563,20 +529,6 @@ dependencies = [
  "want",
 ]
 
-[[package]]
-name = "hyper-rustls"
-version = "0.24.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97"
-dependencies = [
- "futures-util",
- "http",
- "hyper",
- "rustls",
- "tokio",
- "tokio-rustls",
-]
-
 [[package]]
 name = "hyper-timeout"
 version = "0.4.1"
@@ -1063,7 +1015,6 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
- "hyper-rustls",
  "ipnet",
  "js-sys",
  "log",
@@ -1071,37 +1022,18 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls",
- "rustls-pemfile",
  "serde",
  "serde_json",
  "serde_urlencoded",
  "tokio",
- "tokio-rustls",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
- "webpki-roots",
  "winreg",
 ]
 
-[[package]]
-name = "ring"
-version = "0.16.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
-dependencies = [
- "cc",
- "libc",
- "once_cell",
- "spin",
- "untrusted",
- "web-sys",
- "winapi",
-]
-
 [[package]]
 name = "rustc-demangle"
 version = "0.1.23"
@@ -1121,37 +1053,6 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "rustls"
-version = "0.21.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
-dependencies = [
- "log",
- "ring",
- "rustls-webpki",
- "sct",
-]
-
-[[package]]
-name = "rustls-pemfile"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
-dependencies = [
- "base64",
-]
-
-[[package]]
-name = "rustls-webpki"
-version = "0.101.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d"
-dependencies = [
- "ring",
- "untrusted",
-]
-
 [[package]]
 name = "rustversion"
 version = "1.0.14"
@@ -1164,16 +1065,6 @@ version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
 
-[[package]]
-name = "sct"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
-dependencies = [
- "ring",
- "untrusted",
-]
-
 [[package]]
 name = "serde"
 version = "1.0.188"
@@ -1270,12 +1161,6 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "spin"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
-
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -1401,16 +1286,6 @@ dependencies = [
  "syn 2.0.31",
 ]
 
-[[package]]
-name = "tokio-rustls"
-version = "0.24.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
-dependencies = [
- "rustls",
- "tokio",
-]
-
 [[package]]
 name = "tokio-stream"
 version = "0.1.14"
@@ -1443,7 +1318,6 @@ version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
 dependencies = [
- "async-stream",
  "async-trait",
  "axum",
  "base64",
@@ -1458,9 +1332,7 @@ dependencies = [
  "percent-encoding",
  "pin-project",
  "prost",
- "rustls-pemfile",
  "tokio",
- "tokio-rustls",
  "tokio-stream",
  "tower",
  "tower-layer",
@@ -1617,12 +1489,6 @@ dependencies = [
  "tinyvec",
 ]
 
-[[package]]
-name = "untrusted"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
-
 [[package]]
 name = "url"
 version = "2.4.1"
@@ -1737,12 +1603,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "webpki-roots"
-version = "0.25.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
-
 [[package]]
 name = "which"
 version = "4.4.2"

Vagrantfile

@@ -0,0 +1,11 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  config.vm.box = "debian/bullseye64"
+  config.vm.network "forwarded_port", guest: 50012, host: 50012, host_ip: "127.0.0.1"
+end

agent/Cargo.toml

@@ -12,13 +12,13 @@ itertools = "0.11.0"
 nix = "0.27.1"
 once_cell = "1.18.0"
 prost = "0.11.9"
-reqwest = { version = "0.11.18", features = ["blocking", "rustls-tls", "json"], default-features = false }
+reqwest = { version = "0.11.18", features = ["blocking", "json"], default-features = false }
 serde = { version = "1.0.173", features = ["derive"] }
 serde_json = "1.0.103"
 sysinfo = { version = "0.29.2", default-features = false }
 tokio = { version = "1.28.2", features = ["rt-multi-thread", "io-util", "process", "macros", "signal"] }
 tokio-stream = { version = "0.1.14", features = ["net", "sync"] }
-tonic = { version = "0.9.2", features = ["tls"] }
+tonic = { version = "0.9.2" }
 tower-http = { version = "0.4.3", features = ["trace"] }
 tracing = "0.1.37"
 tracing-subscriber = "0.3.17"

app/lib/prymn/servers.ex

@@ -131,7 +131,7 @@ defmodule Prymn.Servers do
     token
     |> Base.encode64()
     |> then(fn token ->
-      "curl -sSfL " <> PrymnWeb.Endpoint.url() <> "/install | sh -s " <> token
+      "wget -O- " <> PrymnWeb.Endpoint.url() <> "/install | sudo sh -s " <> token
     end)
   end
 end

flake.lock

@@ -1,5 +1,48 @@
 {
   "nodes": {
+    "crane": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": [
+          "rust-overlay"
+        ]
+      },
+      "locked": {
+        "lastModified": 1696384830,
+        "narHash": "sha256-j8ZsVqzmj5sOm5MW9cqwQJUZELFFwOislDmqDDEMl6k=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "f2143cd27f8bd09ee4f0121336c65015a2a0a19c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696267196,
+        "narHash": "sha256-AAQ/2sD+0D18bb8hKuEEVpHUYD1GmO2Uh/taFamn6XQ=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "4f910c9827911b1ec2bf26b5a062cd09f8d89f85",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "inputs": {
         "systems": "systems"
@@ -36,6 +79,7 @@
     },
     "root": {
       "inputs": {
+        "crane": "crane",
         "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs",
         "rust-overlay": "rust-overlay"

flake.nix

@@ -9,13 +9,21 @@
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.flake-utils.follows = "flake-utils";
     };
+    crane = {
+      url = "github:ipetkov/crane";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.flake-utils.follows = "flake-utils";
+      inputs.rust-overlay.follows = "rust-overlay";
+    };
   };
 
-  outputs = { self, nixpkgs, flake-utils, rust-overlay }: flake-utils.lib.eachDefaultSystem (system:
+  outputs = { self, nixpkgs, flake-utils, rust-overlay, crane }:
+    flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = import nixpkgs { inherit system overlays; };
         overlays = [ (import ./nix/overlay.nix) (import rust-overlay) ];
         scripts = pkgs.callPackage ./nix/scripts.nix { };
+        buildRust = buildTarget: pkgs.callPackage ./nix/rust.nix { inherit crane buildTarget; };
       in
       {
         devShells.default = with pkgs; mkShell {
@@ -32,5 +40,8 @@
             export PROJECT_ROOT_DIR="$PWD"
           '';
         };
+
+        packages.agent.x86_64 = buildRust "x86_64-unknown-linux-musl";
+        packages.agent.aarch64 = buildRust "aarch64-unknown-linux-musl";
       });
 }

get_prymn.sh

@@ -6,11 +6,11 @@
 
 set -u
 
-ROOT_URL=${ROOT_URL:-"https://static.prymn.net/agent/dist"}
+PRYMN_ROOT_URL=${PRYMN_ROOT_URL:-"https://static.prymn.net/agent"}
 
 main() {
     if [ $# -eq 0 ]; then
-        exit_error "missing command parameters. please make sure you copied the installation command correctly."
+        exit_error "I was not called correctly. Please make sure you copied the installation command correctly."
     fi
 
     check_user_is_root
@@ -24,9 +24,9 @@ main() {
     local dir
     dir="$(mktemp -d)"
     local file="${dir}/prymn_agent"
-    local url="${ROOT_URL}/${arch}/prymn_agent"
+    local url="${PRYMN_ROOT_URL}/${arch}/prymn_agent"
 
-    printf "info: downloading prymn agent...\n" 1>&2
+    info "Download the Prymn agent..."
 
     ensure download "${url}" "${file}"
     ensure chmod u+x "${file}"
@@ -43,7 +43,11 @@ main() {
 
 check_user_is_root() {
     if [ "$(id -u)" -ne 0 ]; then
-        exit_error "this script must be run as the root user."
+        exit_error "You must run me as the root user. For example you may execute:
+
+  wget -O- <url> | sudo sh -s <token>
+
+replace <url> and <token> with the ones provided in the server creation page."
     fi
 }
 
@@ -90,9 +94,7 @@ check_os() {
     esac
 
     local arch clib kernel cputype
-    # TODO: check for musl (probably not be needed because we support distros
+    clib="musl"
-    # that don't use it)
-    clib="gnu"
     kernel="$(uname -s)"
     cputype="$(uname -m)"
 
@@ -124,19 +126,45 @@ check_os() {
 }
 
 download() {
+    local download
+    if command -v curl > /dev/null; then
+        download="curl -sSfL $1 -o $2"
+    elif command -v wget > /dev/null; then
+        download="wget -O $2 $1"
+    else
+        exit_error "No downloader found in your system. Please install either 'wget' or 'curl'."
+    fi
+
     local err
-    err=$(curl -sSfL "$1" -o "$2" 2>&1)
+    err=$("$download" 2>&1)
     if [ -n "$err" ]; then
         exit_error "$err"
     fi
 }
 
+check_downloader() {
+    local downloader
+    if command -v curl > /dev/null; then
+        downloader="curl -sSfL"
+    elif command -v wget > /dev/null; then
+        downloader="wget -O-"
+    else
+        exit_error "No downloader found in your system. Please install either 'wget' or 'curl'."
+    fi
+
+    RETURN_VALUE="${downloader}"
+}
+
+info() {
+    printf "%s\n" "$1" 1>&2
+}
+
 warning() {
-    printf "warning: %s\n" "$1" >&2
+    printf "Warning: %s\n" "$1" >&2
 }
 
 exit_error() {
-    printf "error: %s\n" "$1" >&2
+    printf "Error: %s\n" "$1" >&2
     exit 1
 }
 

nix/rust.nix

@@ -0,0 +1,22 @@
+{ pkgs
+, crane
+, buildTarget ? "x86_64-unknown-linux-musl"
+}:
+
+let
+  craneLib = (crane.mkLib pkgs).overrideToolchain pkgs.rustToolchain;
+  buildInputs = [ pkgs.protobuf ];
+  doCheck = false;
+  src = pkgs.lib.cleanSourceWith {
+    src = craneLib.path ../.;
+    filter = path: type:
+      (craneLib.filterCargoSources path type) ||
+      (builtins.match ".*proto$" path != null);
+  };
+in
+craneLib.buildPackage {
+  inherit src doCheck buildInputs;
+
+  CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static -C linker=rust-lld";
+  CARGO_BUILD_TARGET = buildTarget;
+}