Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Nixfying #536

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
10,529 changes: 10,529 additions & 0 deletions Cargo.lock.nix

Large diffs are not rendered by default.

71 changes: 71 additions & 0 deletions commands.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
{ pkgs ? import <nixpkgs> {}
, prefix ? "glm"
# this default "fetch" is for nix-build but we will pass here the golem-examples
# from flake to make it easier to update
, golem-examples ? pkgs.fetchFromGitHub {
owner = "golemcloud";
repo = "golem-examples";
rev = "5165dcc7e3cbfa09f752caa96a869d284ec169aa";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why this revision? What happens when we update golem-exampels? (we do it quite frequently)

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this revision is just for "testing" this parameter will be overwritten by flake, this is used in nix-build/nix-shell only
flake suppose to be the default use (nix build) look at the comment at the flake inputs to see how to keep update.

hash = "sha256-7uWvvrRpIo4euFgP0TG4PEXENl4+Wgd8ckPpYnAwQbw=";
}
}:
let

commands = pkgs.lib.fix (self: pkgs.lib.mapAttrs pkgs.writeShellScript
{
welcome = ''
${pkgs.figlet}/bin/figlet 'golem dev shell'
echo 'press ${prefix}-<TAB><TAB> to see all the commands'
'';
git = ''${pkgs.git}/bin/git "$@"'';

git-project-path = ''${self.git} rev-parse --show-toplevel'';

add-golem-examples-symlink-info = ''
echo 'this command will symlink the golem-example project into the current folder.'
echo 'we have to do this due to issue with nix not being able to load correctly custom build script dependencies'
'';
add-golem-examples-symlink = ''
ln -sf ${golem-examples} golem-examples
'';
update-golem-examples-from-lock =''
nix flake lock --update-input golem-examples
'';

create-patches = ''
${self.git} diff $(${self.git-project-path})/Cargo.lock $(${self.git-project-path})/golem-cli/Cargo.toml > nixDeps.patch
${self.git} diff $(${self.git-project-path})/golem-client/build.rs > fixOldSyntax.patch
'';
apply-patches = ''
${self.git} apply *.patch
'';
revert-patches = ''
${self.git} apply -R *.patch
'';

build-with-nix-deps-golem-cli = ''
${pkgs.cargo}/bin/cargo build -p golem-cli
'';

update-deps = ''
${self.update-golem-examples-from-lock} && \
${self.apply-patches} && \
${self.add-golem-examples-symlink} && \
${self.build-with-nix-deps-golem-cli} && \
${self.create-patches} && \
${self.revert-patches}
'';
default = self.update-deps;

});
in pkgs.symlinkJoin rec {
name = prefix;
passthru.set = commands;
passthru.bin = pkgs.lib.mapAttrs (name: command: pkgs.runCommand "${prefix}-${name}" {} ''
mkdir -p $out/bin
ln -sf ${command} $out/bin/${
if name == "default" then prefix else prefix+"-"+name
}
'') commands;
paths = pkgs.lib.attrValues passthru.bin;
}
41 changes: 41 additions & 0 deletions default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
{ pkgs? import <nixpkgs> {}
, golem-examples ? pkgs.fetchFromGitHub {
owner = "golemcloud";
repo = "golem-examples";
rev = "5165dcc7e3cbfa09f752caa96a869d284ec169aa";
hash = "sha256-7uWvvrRpIo4euFgP0TG4PEXENl4+Wgd8ckPpYnAwQbw=";
}
}:
pkgs.rustPlatform.buildRustPackage {
pname = "golem-cli";
version = "0.0.98";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is outdated now - who and when would update it? We are not keeping our version numbers anywhere else in our repo but use git tags to do releases.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it can be set to 0.0.0, I guess it will be possible to set it in a shellScript command before build from a tag but that will count as kinda of "side effect" and then you will have issue for people that will want to run:

nix build github:golemcloud/golem

without cloning it.


src = pkgs.lib.cleanSource ./.;

cargoLock = {
lockFileContents = builtins.readFile ./Cargo.lock.nix;
allowBuiltinFetchGit = false;
outputHashes = {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is special about these two dependencies? What if they need to be updated? Or how do we know we need them or need something else here?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not 100% sure why nix could not include this dependencies directly from lock file, I had to include them explicitly to resolve some conflicts.
in order to update them it could be done manually if they are not updated that frequently otherwise it is possible to add another command to update-deps command in commands.nix (dev shell)

"libtest-mimic-0.7.0" = "sha256-xUAyZbti96ky6TFtUjyT6Jx1g0N1gkDPjCMcto5SzxE=";
"cranelift-bforest-0.104.0" = "sha256-veZc4s+OitjBv4ohzzgFdAxLm/J/B5NVy+RXU0hgfwQ=";
};
};
patches = [ ./nixDeps.patch ./fixOldSyntax.patch ];
postPatch = ''
cp -r ${golem-examples} golem-examples
'';

cargoBuildFlags = [ "-p" "golem-cli" ];
nativeBuildInputs = [
pkgs.pkg-config

];

PROTOC = "${pkgs.protobuf}/bin/protoc";

buildInputs = [
pkgs.openssl
];

doCheck = false;
}
15 changes: 15 additions & 0 deletions fixOldSyntax.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
diff --git a/golem-client/build.rs b/golem-client/build.rs
index e93bafa..16eb4c1 100644
--- a/golem-client/build.rs
+++ b/golem-client/build.rs
@@ -27,8 +27,8 @@ fn main() {
)
.unwrap();

- println!("cargo::rerun-if-changed=build.rs");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file tells me that this nix package is trying to build Golem with an older rust version (the :: syntax is since 1.77). We don't want to pin our codebase to an old rust version, and are potentially using new rust features so even though we could change this particular syntax back but the build can break any time if it's not using the same rust version that we are using (which is, always the latest, for now).

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can try to run an update (remove/update flake.lock) and update the dependencies we might wont need this line anymore.

- println!("cargo::rerun-if-changed=openapi/golem-service.yaml");
+ println!("cargo:rerun-if-changed=build.rs");
+ println!("cargo:rerun-if-changed=openapi/golem-service.yaml");
} else {
let crate_yaml_path = Path::new(&manifest_dir).join("openapi/golem-service.yaml");
generate(crate_yaml_path, out_dir);
43 changes: 43 additions & 0 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

37 changes: 37 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
inputs.golem-examples.url = "github:golemcloud/golem-examples";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is golem-examples special for nix, and not our other dependencies? (For example golem-wasm-rpc, golem-wasm-ast, golem-wit, etc)

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had issue building golem-examples when building golem-cli from nix
I think it was more "side effecty" setup.rs then the other projects, it was missing "Inflector" package. I was able to solve it referencing it on the nix build of golem-cli.
update-deps in the dev shell (commands.nix) will update it to latest version

inputs.golem-examples.flake = false;
inputs.nixpkgs.url = "github:NixOS/nixpkgs";

outputs = { self, nixpkgs, golem-examples }:
let system = "x86_64-linux";
pkgs = import nixpkgs { inherit system;};
commands = import ./commands.nix { inherit pkgs golem-examples;};
in
{
# packages.${system}.update = import ./default.nix { inherit pkgs golem-examples; };
packages.${system}.default = import ./default.nix { inherit pkgs golem-examples;
# updateTOML = commands.set.replace-nix-deps-in-tomls;
};
devShells.${system}.default = pkgs.mkShell {
name = "shell";
buildInputs = [
commands

pkgs.sd

pkgs.cargo
pkgs.rustc
pkgs.rustPlatform.cargoSetupHook
pkgs.rustPlatform.maturinBuildHook
pkgs.openssl.dev
pkgs.rustPlatform.bindgenHook
pkgs.pkg-config
pkgs.openssl
];

PROTOC = "${pkgs.protobuf}/bin/protoc";
shellHook = commands.set.welcome;
};
};
}
82 changes: 82 additions & 0 deletions golem-cli/Cargo.toml.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
[package]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is there a copy of our golem-cli cargo file here? Who and when will keep it updated? What is the difference?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the difference between this file and Cargo.toml is the change to of golem examples to be to one that is brought by nix (inputs.golem-examples, and additional missing reference),
I think I had planned to be included in the update-deps command on the commands.nix but i might missed something I could look at it this weekend.

name = "golem-cli"
version = "0.0.0"
edition = "2021"
license = "Apache-2.0"
homepage = "https://golem.cloud/"
repository = "https://github.com/golemcloud/golem"
authors = ["Simon Popugaev <[email protected]>"]
readme = "README.md"
description = "Command line interface for OSS version of Golem. See also golem-cloud-cli."
autotests = false

[[test]]
name = "integration"
path = "tests/main.rs"
harness = false

[dependencies]
golem-client = { path = "../golem-client", version = "0.0.0" }

async-trait = { workspace = true }
chrono = { workspace = true }
clap = { version = "4.4.12", features = ["derive"] }
clap-verbosity-flag = "2.1.1"
derive_more = "0.99.17"
dirs ="5.0.1"
futures-util = { workspace = true }
golem-examples = { path = "../golem-examples" }
golem-wasm-ast = { workspace = true }
golem-wasm-rpc = { workspace = true, features = ["text"] }
golem-wasm-rpc-stubgen = { version = "0.0.22", optional = true }
h2 = "0.3.24"
http = { workspace = true }
hyper = "0.14.28"
indoc = "2.0.4"
itertools = "0.11.0"
native-tls = "0.2.11"
openapiv3 = "2.0.0"
rand = "0.8.5"
reqwest = { version = "0.11.23", features = ["stream", "json", "multipart"] }
serde = { workspace = true }
serde_json = { workspace = true }
serde_yaml = "0.9.29"
strum = "0.25.0"
strum_macros = "0.25.3"
testcontainers-modules = { version = "0.3.2", features = ["postgres", "redis"] }
tokio = { workspace = true }
tokio-tungstenite = { version = "0.20.1", features = ["native-tls"] }
tower = "0.4.13"
tracing = "0.1.40"
tracing-subscriber = "0.3.18"
tungstenite = "0.20.1"
url = { workspace = true }
uuid = { workspace = true }
version-compare = "=0.0.11"
wasm-wave = "=0.4.0"
cli-table = "0.4.7"
textwrap = "0.16.1"

[dev-dependencies]
golem-test-framework = { path = "../golem-test-framework", version = "0.0.0" }
Inflector = "0.11.4"

async-recursion = "1.0.5"
env_logger = "0.11.1"
libtest-mimic = { git = "https://github.com/senia-psm/libtest-mimic.git", branch = "async_and_context_tests" }
log = "0.4.20"
postgres = "0.19.7"
rand = "0.8.5"
redis = "0.24.0"
serde_json_path = "0.6.5"
testcontainers = { version = "0.15.0", features = ["async-trait", "tokio"] }
testcontainers-modules = { version = "0.3.2", features = ["postgres", "redis"] }
tokio-postgres = "0.7.10"
tonic = "0.10.2"
tonic-health = "0.10.2"
regex = "1.10.4"
strip-ansi-escapes = "0.2.0"

[features]
default = ["stubgen"]
stubgen = ["dep:golem-wasm-rpc-stubgen"]
51 changes: 51 additions & 0 deletions nixDeps.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
diff --git a/Cargo.lock b/Cargo.lock
index ee1945a..b33d791 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3130,6 +3130,7 @@ dependencies = [
name = "golem-cli"
version = "0.0.0"
dependencies = [
+ "Inflector",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this dependency added by a patch here?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I had to include it to build correctly golem-examples but I might be wrong I will check it this weekend

"async-recursion",
"async-trait",
"chrono",
@@ -3320,12 +3321,11 @@ dependencies = [

[[package]]
name = "golem-examples"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d97faf7aba9d41c971f625762fa95a993586b7d9389b7ec0ae85c8fdc512bda4"
+version = "0.0.0"
dependencies = [
"Inflector",
"cargo_metadata",
+ "clap 4.5.4",
"copy_dir",
"derive_more",
"dir-diff",
diff --git a/golem-cli/Cargo.toml b/golem-cli/Cargo.toml
index 3a5e0bc..7feb504 100644
--- a/golem-cli/Cargo.toml
+++ b/golem-cli/Cargo.toml
@@ -23,9 +23,9 @@ chrono = { workspace = true }
clap = { version = "4.4.12", features = ["derive"] }
clap-verbosity-flag = "2.1.1"
derive_more = "0.99.17"
-dirs = "5.0.1"
+dirs ="5.0.1"
futures-util = { workspace = true }
-golem-examples = "0.2.1"
+golem-examples = { path = "../golem-examples" }
golem-wasm-ast = { workspace = true }
golem-wasm-rpc = { workspace = true, features = ["text"] }
golem-wasm-rpc-stubgen = { version = "0.0.22", optional = true }
@@ -59,6 +59,7 @@ textwrap = "0.16.1"

[dev-dependencies]
golem-test-framework = { path = "../golem-test-framework", version = "0.0.0" }
+Inflector = "0.11.4"

async-recursion = "1.0.5"
env_logger = "0.11.1"
Loading