Skip to content

Commit 3ba6fd3

Browse files
ttuegelttuegel
authored
Materialize Nix expressions (#2301)
* Remove support for local overrides Local overrides ruin binary caching. * Materialize haskell.nix projects Materialization saves the part of the Nix expression which would be generated every time the project is built. This part tracks the package description and dependencies, so it only changes infrequently. Saving the output of this step (so that we can skip building it) makes the build faster. The drawback of this approach is that we must periodically run with checkMaterialization enabled to detect when the materialization needs to be updated. * Separate test and release workflows * Run release workflow only on push to master * Add nix/rematerialize.sh * Run Test workflow on pull requests only * Add a Stack step to Test workflow
1 parent c160ec5 commit 3ba6fd3

26 files changed

+2269
-94
lines changed

‎.github/workflows/release.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: "Release"
2+
on:
3+
push:
4+
branches:
5+
- master
6+
jobs:
7+
check:
8+
name: 'Check'
9+
runs-on: ubuntu-latest
10+
steps:
11+
- name: Check out code
12+
uses: actions/[email protected]
13+
with:
14+
submodules: recursive
15+
16+
- name: Install Nix
17+
uses: cachix/install-nix-action@v12
18+
with:
19+
extra_nix_config: |
20+
substituters = http://cache.nixos.org https://hydra.iohk.io
21+
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=
22+
23+
- name: Install Cachix
24+
uses: cachix/cachix-action@v8
25+
with:
26+
name: runtimeverification
27+
extraPullNames: 'kore'
28+
skipPush: true
29+
30+
- name: Check materialization
31+
run: nix-build --arg checkMaterialization true -A project.stack-nix
32+
33+
release:
34+
name: 'Release'
35+
runs-on: ubuntu-latest
36+
steps:
37+
- name: Check out code
38+
uses: actions/[email protected]
39+
with:
40+
submodules: recursive
41+
42+
- name: Install Nix
43+
uses: cachix/install-nix-action@v12
44+
with:
45+
extra_nix_config: |
46+
substituters = http://cache.nixos.org https://hydra.iohk.io
47+
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=
48+
49+
- name: Install Cachix
50+
uses: cachix/cachix-action@v8
51+
with:
52+
name: runtimeverification
53+
signingKey: '${{ secrets.RUNTIMEVERIFICATION_CACHIX_SIGNING_KEY }}'
54+
extraPullNames: 'kore'
55+
56+
- name: Build
57+
run: nix-build -A kore -A project.kore.checks

‎.github/workflows/test.yml‎

Lines changed: 108 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,116 @@
11
name: "Test"
22
on:
33
pull_request:
4-
push:
54
jobs:
6-
test-nix:
5+
nix:
6+
name: 'Nix'
77
runs-on: ubuntu-latest
88
steps:
9-
- uses: actions/[email protected]
10-
with:
11-
submodules: recursive
12-
- uses: cachix/install-nix-action@v12
13-
- uses: cachix/cachix-action@v8
14-
with:
15-
name: runtimeverification
16-
signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
17-
- run: nix-build -A kore -A project.kore.checks
18-
- run: nix-shell --run "echo OK"
19-
20-
test-cabal:
9+
- name: Check out code
10+
uses: actions/[email protected]
11+
with:
12+
submodules: recursive
13+
# Avoid cloning a detached-HEAD repository on pull_request events.
14+
# git-auto-commit-action (below) needs this to find the original
15+
# branch where it should push the changes.
16+
ref: ${{ github.head_ref }}
17+
18+
- name: Install Nix
19+
uses: cachix/install-nix-action@v12
20+
with:
21+
extra_nix_config: |
22+
substituters = http://cache.nixos.org https://hydra.iohk.io
23+
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=
24+
25+
- name: Install Cachix
26+
uses: cachix/cachix-action@v8
27+
with:
28+
name: kore
29+
signingKey: '${{ secrets.KORE_CACHIX_SIGNING_KEY }}'
30+
31+
- name: Materialize
32+
run: ./nix/rematerialize.sh
33+
34+
- name: Commit changes
35+
uses: stefanzweifel/[email protected]
36+
with:
37+
commit_message: 'Materialize Nix expressions'
38+
file_pattern: 'nix/'
39+
40+
- name: Build
41+
run: nix-build -A kore
42+
43+
- name: Run unit tests
44+
run: nix-build -A project.kore.checks
45+
46+
- name: Check shell
47+
run: nix-shell --run "echo OK"
48+
49+
cabal:
50+
name: 'Cabal'
2151
runs-on: ubuntu-latest
2252
steps:
23-
- uses: actions/[email protected]
24-
with:
25-
submodules: recursive
26-
27-
- uses: actions/[email protected]
28-
id: setup-haskell-cabal
29-
with:
30-
ghc-version: "8.10.1"
31-
cabal-version: "3.2"
32-
33-
- name: Cache Cabal store
34-
uses: actions/cache@v2
35-
with:
36-
path: ${{ steps.setup-haskell-cabal.outputs.cabal-store }}
37-
key: ${{ runner.os }}-ghc-8.10.1-${{ hashFiles('cabal.project.freeze') }}
38-
39-
- name: Build project
40-
run: cabal v2-build --enable-tests --enable-benchmarks all
41-
42-
- name: Configure with profiling
43-
run: cabal v2-configure --enable-profiling -f-threaded
53+
- name: Install prerequisites
54+
run: |
55+
sudo apt install --yes z3
56+
57+
- uses: actions/[email protected]
58+
with:
59+
submodules: recursive
60+
61+
- uses: actions/[email protected]
62+
id: setup-haskell-cabal
63+
with:
64+
ghc-version: "8.10.1"
65+
cabal-version: "3.2"
66+
67+
- name: Cache Cabal store
68+
uses: actions/cache@v2
69+
with:
70+
path: ${{ steps.setup-haskell-cabal.outputs.cabal-store }}
71+
key: ${{ runner.os }}-ghc-8.10.1-${{ hashFiles('cabal.project.freeze') }}
72+
73+
- name: Build
74+
run: cabal v2-build --enable-tests --enable-benchmarks all
75+
76+
- name: Run unit tests
77+
run: cabal v2-test --test-show-details=direct all
78+
79+
- name: Configure with profiling
80+
run: cabal v2-configure --enable-profiling -f-threaded
81+
82+
stack:
83+
name: 'Stack'
84+
runs-on: ubuntu-latest
85+
steps:
86+
- name: Install prerequisites
87+
run: |
88+
sudo apt install --yes z3
89+
90+
- uses: actions/[email protected]
91+
with:
92+
submodules: recursive
93+
94+
- uses: actions/[email protected]
95+
id: setup-haskell-stack
96+
with:
97+
enable-stack: true
98+
stack-no-global: true
99+
stack-setup-ghc: true
100+
101+
- name: Locate Stack root
102+
id: locate-stack-root
103+
run: |
104+
echo "::set-output name=stack-root::$(stack path --stack-root)"
105+
106+
- name: Cache Stack root
107+
uses: actions/cache@v2
108+
with:
109+
path: ${{ steps.locate-stack-root.outputs.stack-root }}
110+
key: ${{ runner.os }}-ghc-8.10.1-${{ hashFiles('stack.yaml.lock') }}
111+
112+
- name: Build project
113+
run: stack build --pedantic
114+
115+
- name: Run unit tests
116+
run: stack test --pedantic

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
/.build/*
22
/evm-semantics
33

4-
*.cache
54
*.class
65
*.iml
76
*.ipr

‎README.md‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -44,23 +44,27 @@ If using `cabal`, version 3.0 or later is recommended.
4444

4545
## Developing
4646

47-
Developers will require all the dependencies listed above.
48-
We also recommend (but not require!) the following dependencies.
47+
Developers will require all the dependencies listed above,
48+
in addition to the requirements and recommendations below.
49+
50+
### Required dependencies
51+
52+
For integration testing, we require:
53+
54+
- GNU [make]
55+
- The [K Framework] frontend, or [curl] to fetch an appropriate version.
56+
The frontend has other dependencies, most notably a Java runtime.
57+
58+
### Recommended dependencies
4959

5060
For setting up a development environment, we recommend:
5161

5262
- [direnv] to make the project's tools available in shells and editors.
53-
- [ghcide] or [haskell-ide-engine], [language servers] for Haskell that are
63+
- [ghcide] or [haskell-language-server], [language servers] for Haskell that are
5464
compatible with most editors. See instructions
5565
[below](#running-a-language-server) to run a language server.
5666
- [hlint] and [stylish-haskell] for compliance with project guidelines.
5767

58-
For integration testing, we also recommend:
59-
60-
- GNU [make]
61-
- The [K Framework] frontend, or [curl] to fetch an appropriate version.
62-
The frontend has other dependencies, most notably a Java runtime.
63-
6468
### Running a language server
6569

6670
To run a language server, developers will need to activate the appropriate
@@ -85,9 +89,18 @@ cabal build --enable-tests --enable-benchmarks --only-dependencies kore
8589

8690
### Developing with Nix
8791

88-
For developers so inclined, we provide a `shell.nix` expression with a suitable
89-
development environment and a binary cache at [kore.cachix.org]. The development
90-
environment is intended to be used with `nix-shell` and `cabal`.
92+
We provide a `shell.nix` expression with a suitable development environment and
93+
a binary cache at [kore.cachix.org]. The development environment is intended to
94+
be used with `nix-shell` and `cabal`.
95+
96+
When the `.cabal` package description file changes, run:
97+
98+
```.sh
99+
# Requires Nix to be installed.
100+
./nix/rematerialize.sh
101+
```
102+
103+
This script is also run by an automatic workflow.
91104

92105

93106
[git]: https://git-scm.com/
@@ -98,8 +111,9 @@ environment is intended to be used with `nix-shell` and `cabal`.
98111
[make]: https://www.gnu.org/software/make/
99112
[direnv]: https://github.com/direnv/direnv
100113
[ghcide]: https://github.com/digital-asset/ghcide
101-
[haskell-ide-engine]: https://github.com/haskell/haskell-ide-engine
114+
[haskell-language-server]: https://github.com/haskell/haskell-language-server
102115
[language servers]: https://langserver.org/
103116
[hlint]: https://github.com/ndmitchell/hlint
104117
[stylish-haskell]: https://github.com/jaspervdj/stylish-haskell
105118
[kore.cachix.org]: https://kore.cachix.org/
119+
[Nix]: https://nixos.org

‎default.nix‎

Lines changed: 28 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{ profiling ? false
22
, release ? false
33
, threaded ? !profiling
4+
, checkMaterialization ? false
45
}:
56

67
let
@@ -14,38 +15,34 @@ let
1415
in import haskell-nix.sources.nixpkgs-2003 args;
1516
inherit (pkgs) lib;
1617

17-
local =
18-
if builtins.pathExists ./local.nix
19-
then import ./local.nix { inherit default; }
20-
else x: x;
21-
22-
project =
23-
(args: pkgs.haskell-nix.stackProject (local args)) {
24-
src = pkgs.haskell-nix.haskellLib.cleanGit { name = "kore"; src = ./.; };
25-
modules = [
26-
{
27-
# package *
28-
enableLibraryProfiling = true;
29-
profilingDetail = "none";
30-
# package kore
31-
packages.kore = {
32-
flags = {
33-
inherit release threaded;
34-
};
35-
enableLibraryProfiling = profiling;
36-
enableExecutableProfiling = profiling;
37-
profilingDetail = "toplevel-functions";
38-
39-
# Add Z3 to PATH for unit tests.
40-
components.tests.kore-test.preCheck = ''
41-
export PATH="$PATH''${PATH:+:}${lib.getBin pkgs.z3}/bin"
42-
'';
18+
project = pkgs.haskell-nix.stackProject {
19+
src = pkgs.haskell-nix.haskellLib.cleanGit { name = "kore"; src = ./.; };
20+
inherit checkMaterialization;
21+
materialized = ./nix/kore.nix.d;
22+
modules = [
23+
{
24+
# package *
25+
enableLibraryProfiling = true;
26+
profilingDetail = "none";
27+
# package kore
28+
packages.kore = {
29+
flags = {
30+
inherit release threaded;
4331
};
44-
}
45-
];
46-
};
47-
48-
shell = import ./shell.nix { inherit default; };
32+
enableLibraryProfiling = profiling;
33+
enableExecutableProfiling = profiling;
34+
profilingDetail = "toplevel-functions";
35+
36+
# Add Z3 to PATH for unit tests.
37+
components.tests.kore-test.preCheck = ''
38+
export PATH="$PATH''${PATH:+:}${lib.getBin pkgs.z3}/bin"
39+
'';
40+
};
41+
}
42+
];
43+
};
44+
45+
shell = import ./shell.nix { inherit default checkMaterialization; };
4946

5047
version = project.kore.components.exes.kore-exec.version;
5148

‎kore/kore.cabal‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ description: Please see the [README](README.md) file.
1010
category: Language
1111
homepage: https://github.com/kframework/kore#readme
1212
bug-reports: https://github.com/kframework/kore/issues
13-
author: Virgil Serbanuta
14-
maintainer: virgil.serbanuta@runtimeverification.com
13+
author: Runtime Verification Inc
14+
maintainer: thomas.tuegel@runtimeverification.com
1515
copyright: 2018-2020 Runtime Verification Inc
1616
license: NCSA
1717
license-file: LICENSE

‎kore/package.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ version: 0.35.0.0
33
github: "kframework/kore"
44
license: NCSA
55
license-file: LICENSE
6-
author: "Virgil Serbanuta"
7-
maintainer: "virgil.serbanuta@runtimeverification.com"
6+
author: "Runtime Verification Inc"
7+
maintainer: "thomas.tuegel@runtimeverification.com"
88
copyright: "2018-2020 Runtime Verification Inc"
99
category: Language
1010

‎nix/ghcide.nix.d/.stack-to-nix.cache‎

Whitespace-only changes.

0 commit comments

Comments
 (0)