Build with ghc(js) 9.8.2 + 9.10.1 #474

ymeister · 2024-08-06T02:42:50Z

Needs:

For haskell.nix:
default.nix:

let deps = {
      "haskell.nix" = builtins.fetchTarball {
        url = "https://github.com/input-output-hk/haskell.nix/archive/d8c50dcaf3d3d589829ee9be9d5dba8279b8cc59.tar.gz";
        sha256 = "0a5hgryz6nszmy67yf1aks399h2aw0nj845518c4prs5c6ns1z7p";
      };
      patch = pkgs.fetchFromGitHub {
        owner = "ymeister";
        repo = "patch";
        rev = "2170c364450d5827a21dbcd817131d5def6e4767";
        sha256 = "0cnk89h3z0xkfa7jyz9ihycvpa0ak8kyslfl7labkwf6qi3qh80s";
      };
      reflex = pkgs.fetchFromGitHub {
        owner = "ymeister";
        repo = "reflex";
        rev = "844d88d10cbf0db8ad8677a9c72f6a10e811c0f4";
        sha256 = "013iaa4b9d18d8cbszrmp7h153yljsg05b28fblkpyra5ss010qh";
      };
    };

    haskellNix = import deps."haskell.nix" {};

    # Import nixpkgs and pass the haskell.nix provided nixpkgsArgs
    pkgs = import
      # haskell.nix provides access to the nixpkgs pins which are used by our CI,
      # hence you will be more likely to get cache hits when using these.
      # But you can also just use your own, e.g. '<nixpkgs>'.
      haskellNix.sources.nixpkgs-unstable
      # These arguments passed to nixpkgs, include some patches and also
      # the haskell.nix functionality itself as an overlay.
      haskellNix.nixpkgsArgs;

    source-repository-packages = packages:
      builtins.zipAttrsWith
        (k: vs:
          if k == "cabalProjectLocal" then pkgs.lib.strings.concatStringsSep "\n" vs
          else builtins.zipAttrsWith (_: pkgs.lib.lists.last) vs
        )
        (pkgs.lib.lists.forEach packages (p:
          let input = builtins.unsafeDiscardStringContext p;
          in {
            inputMap."${input}" = { name = builtins.baseNameOf p; outPath = p; rev = "HEAD"; };
            cabalProjectLocal = ''
              source-repository-package
                type: git
                location: ${input}
                tag: HEAD
            '';
          }
        ));

    import-cabal-project = dir: file:
      let path = dir + "/${file}";
          content = ''
            -- ${path}
            ${builtins.readFile path}
          '';
          lines = pkgs.lib.strings.splitString "\n" content;
      in pkgs.lib.strings.concatStringsSep "\n" (
          pkgs.lib.lists.forEach lines (line:
            if pkgs.lib.strings.hasPrefix "import: " line
            then import-cabal-project dir (pkgs.lib.strings.removePrefix "import: " line)
            else line
          )
      );

    haskellDeps = source-repository-packages [
      deps.reflex
      deps.patch
    ];

    project = pkgs: pkgs.haskell-nix.project {
      src = ./.;

      inherit (haskellDeps) inputMap cabalProjectLocal;
      cabalProject = import-cabal-project ./. "cabal.project";

      shell.withHaddock = if pkgs.stdenv.hostPlatform.isGhcjs then false else true;

      modules = [({ pkgs, lib, ... }: {
        packages.reflex-dom-core.components.tests = {
          gc.buildable = lib.mkForce false;
          hydration.buildable = lib.mkForce false;
          #gc.build-tools = [ pkgs.chromium ];
          #hydration.build-tools = [ pkgs.chromium ];
        };
      })];

      compiler-nix-name = "ghc910";
    };

in {
  ghc = project pkgs;
  ghc-js = project pkgs.pkgsCross.ghcjs;
}

shell.nix:

let projects = import ./default.nix;
    project = projects.ghc;
    pkgs = project.pkgs;
in project.shellFor {
  tools = {
    cabal = "latest";
  };

  buildInputs = with pkgs; [
    nodejs-slim_latest
  ];

  # Sellect cross compilers to include.
  crossPlatforms = ps: with ps; [
    ghcjs # Adds support for `javascript-unknown-ghcjs-cabal build` in the shell
  ];

  shellHook = ''
    function setup_cabal_to_nix {
      cabal_project_drv="$(nix-store -q --references "${project.plan-nix.drvPath}" | grep ".*-cabal.project.drv" - | head -n 1)"
      [ ! -z "$cabal_project_drv" ] && cabal_project_out="$(nix-store -q --outputs "$cabal_project_drv" | head -n 1)"
      [ ! -z "$cabal_project_out" ] && \cp -f "$cabal_project_out" cabal.project.local && chmod u+w cabal.project.local

      if [ -f cabal.project.local ]; then
        mkdir -p .nix
        store_paths="$(cat cabal.project.local | sed -n 's#.*\(/nix/store/.*\)#\1#p')"
        echo "$store_paths" | while read store_path; do
          target=".nix/$(echo "$store_path" | sed -n 's#/nix/store/\(.*\)#\1#p' | sed 's#/#-#g')"
          [ ! -e "$target" ] && cp -rf "$store_path" "$target" && chmod u+w -R "$target"
          sed -i "s#$store_path#$(readlink -f "$target")#g" cabal.project.local
        done
      fi
    }

    setup_cabal_to_nix 1> /dev/null
  '';
}

For nixpkgs:

From patch PR:

When building on ghc js, requires single-core building (https://gitlab.haskell.org/ghc/ghc/-/issues/25083#note_578275).

For nix:
patch = haskellLib.overrideCabal (drv: { enableParallelBuilding = false; }) super.patch;

Fix os-string (NixOS/nixpkgs#332123):

os-string = null;

To build a project depending on reflex-dom using haskell.nix:
default.nix:

let deps = {
      "haskell.nix" = builtins.fetchTarball {
        url = "https://github.com/input-output-hk/haskell.nix/archive/d8c50dcaf3d3d589829ee9be9d5dba8279b8cc59.tar.gz";
        sha256 = "0a5hgryz6nszmy67yf1aks399h2aw0nj845518c4prs5c6ns1z7p";
      };
      patch = pkgs.fetchFromGitHub {
        owner = "ymeister";
        repo = "patch";
        rev = "2170c364450d5827a21dbcd817131d5def6e4767";
        sha256 = "0cnk89h3z0xkfa7jyz9ihycvpa0ak8kyslfl7labkwf6qi3qh80s";
      };
      reflex = pkgs.fetchFromGitHub {
        owner = "ymeister";
        repo = "reflex";
        rev = "844d88d10cbf0db8ad8677a9c72f6a10e811c0f4";
        sha256 = "013iaa4b9d18d8cbszrmp7h153yljsg05b28fblkpyra5ss010qh";
      };
      reflex-dom = pkgs.fetchFromGitHub {
        owner = "ymeister";
        repo = "reflex-dom";
        rev = "ede863e98247deb8268f15d663292eb8a714437b";
        sha256 = "1z51gxc77nmnw6ylzkmyqizb4xhnvl6971wi0anm78i81n5aw495";
      };
    };

    haskellNix = import deps."haskell.nix" {};

    # Import nixpkgs and pass the haskell.nix provided nixpkgsArgs
    pkgs = import
      # haskell.nix provides access to the nixpkgs pins which are used by our CI,
      # hence you will be more likely to get cache hits when using these.
      # But you can also just use your own, e.g. '<nixpkgs>'.
      haskellNix.sources.nixpkgs-unstable
      # These arguments passed to nixpkgs, include some patches and also
      # the haskell.nix functionality itself as an overlay.
      haskellNix.nixpkgsArgs;

    source-repository-packages = packages:
      builtins.zipAttrsWith
        (k: vs:
          if k == "cabalProjectLocal" then pkgs.lib.strings.concatStringsSep "\n" vs
          else builtins.zipAttrsWith (_: pkgs.lib.lists.last) vs
        )
        (pkgs.lib.lists.forEach packages (p:
          let input = builtins.unsafeDiscardStringContext p;
          in {
            inputMap."${input}" = { name = builtins.baseNameOf p; outPath = p; rev = "HEAD"; };
            cabalProjectLocal = ''
              source-repository-package
                type: git
                location: ${input}
                tag: HEAD
            '';
          }
        ));

    import-cabal-project = dir: file:
      let path = dir + "/${file}";
          content = ''
            -- ${path}
            ${builtins.readFile path}
          '';
          lines = pkgs.lib.strings.splitString "\n" content;
      in pkgs.lib.strings.concatStringsSep "\n" (
          pkgs.lib.lists.forEach lines (line:
            if pkgs.lib.strings.hasPrefix "import: " line
            then import-cabal-project dir (pkgs.lib.strings.removePrefix "import: " line)
            else line
          )
      );

    haskellDeps = source-repository-packages [
      (deps.reflex-dom + "/reflex-dom")
      (deps.reflex-dom + "/reflex-dom-core")
      deps.reflex
      deps.patch
    ];

    project = pkgs: pkgs.haskell-nix.project {
      src = ./.;

      inherit (haskellDeps) inputMap;
      cabalProject = import-cabal-project ./. "cabal.project";
      cabalProjectLocal = ''
        ${import-cabal-project deps.reflex-dom "cabal.dependencies.project"}

        ${haskellDeps.cabalProjectLocal}

        if arch(javascript)
          extra-packages: ghci
      '';

      shell.withHaddock = if pkgs.stdenv.hostPlatform.isGhcjs then false else true;

      modules = [({ pkgs, lib, ... }: {
        packages.reflex-dom-core.components.tests = {
          gc.buildable = lib.mkForce false;
          hydration.buildable = lib.mkForce false;
          #gc.build-tools = [ pkgs.chromium ];
          #hydration.build-tools = [ pkgs.chromium ];
        };
      })];

      compiler-nix-name = "ghc910";
    };

in {
  ghc = project pkgs;
  ghc-js = project pkgs.pkgsCross.ghcjs;
}

shell.nix:

let projects = import ./default.nix;
    project = projects.ghc;
    pkgs = project.pkgs;
in project.shellFor {
  tools = {
    cabal = "latest";
  };

  buildInputs = with pkgs; [
    nodejs-slim_latest
  ];

  # Sellect cross compilers to include.
  crossPlatforms = ps: with ps; [
    ghcjs # Adds support for `javascript-unknown-ghcjs-cabal build` in the shell
  ];

  shellHook = ''
    function setup_cabal_to_nix {
      cabal_project_drv="$(nix-store -q --references "${project.plan-nix.drvPath}" | grep ".*-cabal.project.drv" - | head -n 1)"
      [ ! -z "$cabal_project_drv" ] && cabal_project_out="$(nix-store -q --outputs "$cabal_project_drv" | head -n 1)"
      [ ! -z "$cabal_project_out" ] && \cp -f "$cabal_project_out" cabal.project.local && chmod u+w cabal.project.local

      if [ -f cabal.project.local ]; then
        mkdir -p .nix
        store_paths="$(cat cabal.project.local | sed -n 's#.*\(/nix/store/.*\)#\1#p')"
        echo "$store_paths" | while read store_path; do
          target=".nix/$(echo "$store_path" | sed -n 's#/nix/store/\(.*\)#\1#p' | sed 's#/#-#g')"
          [ ! -e "$target" ] && cp -rf "$store_path" "$target" && chmod u+w -R "$target"
          sed -i "s#$store_path#$(readlink -f "$target")#g" cabal.project.local
        done
      fi
    }

    setup_cabal_to_nix 1> /dev/null
  '';
}

ymeister · 2024-08-07T07:57:52Z

Needs reflex-frp/reflex#502 to be in hackage to pass the CI.

odr · 2024-08-11T06:54:30Z

How to check it?
I took default.nix and shell.nix from the end of PR description. Then I run cabal init and add reflex-dom into cabal deps. Also add cabal.project.
Next, I open nix-shell (successfully, took 2h), run javascript-unknown-ghcjs-cabal build and get (similar to this):

src/Data/Patch/IntMap.hs:1:1: error: [GHC-87897]
    Exception when trying to run compile-time code:
      getBin: Unknown encoding for constructor
    Code: makeWrapped ''PatchIntMap

Is there a way to build now?

ymeister · 2024-08-11T18:25:03Z

@odr

How to check it? I took default.nix and shell.nix from the end of PR description. Then I run cabal init and add reflex-dom into cabal deps. Also add cabal.project. Next, I open nix-shell (successfully, took 2h), run javascript-unknown-ghcjs-cabal build and get (similar to this):
src/Data/Patch/IntMap.hs:1:1: error: [GHC-87897]
    Exception when trying to run compile-time code:
      getBin: Unknown encoding for constructor
    Code: makeWrapped ''PatchIntMap
Is there a way to build now?

~~Hmm, nix-shell being successful means that patch should have already been compiled successfully, both for ghc and ghc-js.~~ If it's the same problem as in https://gitlab.haskell.org/ghc/ghc/-/issues/25083 , then compiling with javascript-unknown-ghcjs-cabal build -j1 should resolve the issue (although I doubt it's the case). The fastest and most reliable way for me to figure out what's wrong with your case would be if you could give me some stripped down version of the project you're trying to build, be it via repo, branch or sending me an archive.

P.S. Make sure you're using the latest default.nix/shell.nix from PR description. There were some edits done to it.

odr · 2024-08-12T10:14:20Z

I'll share code little bit later. Now I just tried -j1 and add -v3. Now an error:

...
[ 4 of 12] Compiling Data.Patch.IntMap ( src/Data/Patch/IntMap.hs, dist/build/Data/Patch/IntMap.o )
ghc-interp.js: Message not supported with the JavaScript interpreter: InitLinker
CallStack (from HasCallStack):
  error, called at libraries/ghci/GHCi/Run.hs:142:20 in ghci-9.10.1-inplace:GHCi.Run
HasCallStack backtrace:
  collectBacktraces, called at libraries/ghc-internal/src/GHC/Internal/Exception.hs:92:13 in ghc-internal:GHC.Internal.Exception
  toExceptionWithBacktrace, called at libraries/ghc-internal/src/GHC/Internal/Exception.hs:128:3 in ghc-internal:GHC.Internal.Exception

src/Data/Patch/IntMap.hs:1:1: error: [GHC-87897]
    Exception when trying to run compile-time code:
      External interpreter terminated (1)
    Code: makeWrapped ''PatchIntMap
  |
1 | {-# LANGUAGE CPP #-}
  | ^
...

odr · 2024-08-14T18:40:57Z

@ymeister My project: https://github.com/odr/reflex-js. It is empty. I just added default.nix and shell.nix, then create .cabal and cabal.project and add reflex-dom into dependencies.
I still get an errors when build:

[nix-shell:~/git/reflex-js]$ javascript-unknown-ghcjs-cabal build    
Build profile: -w ghc-9.10.1 -O1
In order, the following will be built (use -v for more details):
 - patch-0.0.8.2 (lib) (requires build)

...

[ 4 of 12] Compiling Data.Patch.IntMap ( src/Data/Patch/IntMap.hs, dist/build/Data/Patch/IntMap.o )
src/Data/Patch/IntMap.hs:1:1: error: [GHC-87897]
    Exception when trying to run compile-time code:
      {handle: <file descriptor: 35>}: GHCi.Message.remoteCall: end of file
    Code: makeWrapped ''PatchIntMap
  |
1 | {-# LANGUAGE CPP #-}
  | ^

...
Error: [Cabal-7125]
Failed to build patch-0.0.8.2 (which is required by exe:reflex-js from reflex-js-0.1.0.0). See the build log above for details.



[nix-shell:~/git/reflex-js]$ javascript-unknown-ghcjs-cabal build -j1
Build profile: -w ghc-9.10.1 -O1
In order, the following will be built (use -v for more details):
 - patch-0.0.8.2 (lib) (requires build)

...

[ 4 of 12] Compiling Data.Patch.IntMap ( src/Data/Patch/IntMap.hs, dist/build/Data/Patch/IntMap.o )
ghc-interp.js: Message not supported with the JavaScript interpreter: InitLinker
CallStack (from HasCallStack):
  error, called at libraries/ghci/GHCi/Run.hs:142:20 in ghci-9.10.1-inplace:GHCi.Run
HasCallStack backtrace:
  collectBacktraces, called at libraries/ghc-internal/src/GHC/Internal/Exception.hs:92:13 in ghc-internal:GHC.Internal.Exception
  toExceptionWithBacktrace, called at libraries/ghc-internal/src/GHC/Internal/Exception.hs:128:3 in ghc-internal:GHC.Internal.Exception

src/Data/Patch/IntMap.hs:1:1: error: [GHC-87897]
    Exception when trying to run compile-time code:
      External interpreter terminated (1)
    Code: makeWrapped ''PatchIntMap
  |
1 | {-# LANGUAGE CPP #-}
  | ^

...

Error: [Cabal-7125]
Failed to build patch-0.0.8.2 (which is required by exe:reflex-js from reflex-js-0.1.0.0).

ymeister · 2024-08-14T19:56:38Z

@odr Thank you for providing an example! That said, I did not encounter any problem building it with neither native ghc nor ghc-js. Could you please try deleting dist and/or dist-newstyle directories generated by cabal build, and try to build it again?

odr · 2024-08-14T21:14:11Z

@odr Thank you for providing an example! That said, I did not encounter any problem building it with neither native ghc nor ghc-js. Could you please try deleting dist and/or dist-newstyle directories generated by cabal build, and try to build it again?

I removed dist-newstyle and repeat with -j1. The same...

How do you build literally? From nix-shell?

I'll ask my colleague to try this tomorrow...

ymeister · 2024-08-14T21:38:02Z

@odr

I literally just:

git clone https://github.com/odr/reflex-js.git
cd reflex-js
nix-shell -v
javascript-unknown-ghcjs-cabal build

That's all. My system is running on nixos-unstable (rev: cb9a96f23c491c081b38eab96d22fa958043c9fa), if that is of any help. Although, the whole thing about nix is reproducibility and all the project dependencies have been pinned, so I don't know what's interfering with your side. Your error messages say something about errors with JS interpreter, maybe you have nodejs installed on your system that somehow interferes with something? Although the nodejs_slim_latest from shell.nix should have overrode this.

odr · 2024-08-15T10:03:00Z

@ymeister
I checked my node.js and don't see any problem:

[nix-shell:~/git/reflex-js]$ which node
/nix/store/9ficrfvgnm4q91wgw3bmpfd34dyg6bbz-nodejs-slim-22.3.0/bin/node
[nix-shell:~/git/reflex-js]$ which nodejs
/usr/local/bin/nodejs
[nix-shell:~/git/reflex-js]$ ls -la /usr/local/bin/nodejs 
lrwxrwxrwx 1 root root 19 Aug 15 12:20 /usr/local/bin/nodejs -> /usr/local/bin/node
[nix-shell:~/git/reflex-js]$ node --version
v22.3.0
[nix-shell:~/git/reflex-js]$ nodejs --version
v22.6.0

FI: I am on Ubuntu-22.04. AMD Ryzen 7 4800H, cores=8, threads=16

I also tried to run nix-shell --pure (with adding git which nix into buildInputs) but result is the same.

odr · 2024-08-26T12:31:22Z

@ymeister I built it without problems on the new laptop! There is Ubuntu 24.04, no nodejs installed, multi-user nix setup (on the old one I had single-user setup).
So I am unblocked now but I heard about problems with build from others (e.g. on MacOS)...

alexfmpe and others added 16 commits August 1, 2024 03:05

Fix build of reflex-dom-test-selenium

c4fbd16

Build with ghc 9.0

e1c3614

Build with ghc 9.2

9ab4277

Build with ghc 9.4

c3eaab1

Build with ghc 9.6

a73705e

Disable eta reduce hint

8068141

Build with ghc 9.8

c772c80

Fix build for ghc-9.6

6b53f8a

Make reflex-dom-core buildable on recent ghc js backend

65c64f0

Support new js backend syntax

cfa9ad2

Remove JavaScriptFFI pragma

69477d8

Make reflex-dom buildable on ghc js backend

9f71670

Use Type from Data.Kind instead of *

29918d7

Cleanup warnings

f269ed8

Build with ghc(js)-9.10

65adc45

Move project dependecies into a separate project file for transitivity

b69b759

ymeister changed the title ~~Build with ghc(js)-9.10~~ Build with ghc(js) 9.8.2 + 9.10.1 Aug 7, 2024

ymeister requested review from ryantrinkle and ali-abrar August 7, 2024 08:14

ymeister force-pushed the ghc(js)-9.10 branch from 575af71 to 81b9fc7 Compare August 14, 2024 04:20

ymeister added 2 commits August 14, 2024 08:22

Update bounds with 'cabal-bounds' & recover backwards compatibility

3eac6a8

Extend CI

546989b

ymeister force-pushed the ghc(js)-9.10 branch from 81b9fc7 to 546989b Compare August 14, 2024 04:22

ymeister mentioned this pull request Aug 18, 2024

Build with ghc(js) 9.8.2 + 9.10.1 obsidiansystems/obelisk#1093

Open

6 tasks

Fix ghcjs-dom-javascript

ede863e

Fix foreign JS

e43e052

alexfmpe mentioned this pull request Nov 22, 2024

Somewhat modernize CI version range #479

Merged

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Build with ghc(js) 9.8.2 + 9.10.1 #474

Build with ghc(js) 9.8.2 + 9.10.1 #474

ymeister commented Aug 6, 2024 •

edited

Loading

ymeister commented Aug 7, 2024

odr commented Aug 11, 2024

ymeister commented Aug 11, 2024 •

edited

Loading

odr commented Aug 12, 2024

odr commented Aug 14, 2024

ymeister commented Aug 14, 2024

odr commented Aug 14, 2024 •

edited

Loading

ymeister commented Aug 14, 2024 •

edited

Loading

odr commented Aug 15, 2024

odr commented Aug 26, 2024

Build with ghc(js) 9.8.2 + 9.10.1 #474

Are you sure you want to change the base?

Build with ghc(js) 9.8.2 + 9.10.1 #474

Conversation

ymeister commented Aug 6, 2024 • edited Loading

ymeister commented Aug 7, 2024

odr commented Aug 11, 2024

ymeister commented Aug 11, 2024 • edited Loading

odr commented Aug 12, 2024

odr commented Aug 14, 2024

ymeister commented Aug 14, 2024

odr commented Aug 14, 2024 • edited Loading

ymeister commented Aug 14, 2024 • edited Loading

odr commented Aug 15, 2024

odr commented Aug 26, 2024

ymeister commented Aug 6, 2024 •

edited

Loading

ymeister commented Aug 11, 2024 •

edited

Loading

odr commented Aug 14, 2024 •

edited

Loading

ymeister commented Aug 14, 2024 •

edited

Loading