Merge pull request #7117 from phadej/release-py-from-sdists

release.py builds from sdist tarballs

.github/workflows/artifacts.yml

@@ -26,11 +26,11 @@ jobs:
           add-apt-repository ppa:deadsnakes/ppa
           apt-get update
           apt-get install -y python3.8
-      - name: Update Hackage index
-        run: cabal v2-update
       - uses: actions/checkout@v2
       - name: Package project
-        run: python3.8 release.py
+        run: |
+          cabal v2-sdist --builddir=sdist Cabal cabal-install
+          python3.8 release.py --tarlib=sdist/sdist/Cabal-*.tar.gz --tarexe=sdist/sdist/cabal-install-*.tar.gz
       - uses: actions/upload-artifact@v2
         with:
           name: cabal-linux
@@ -61,11 +61,11 @@ jobs:
           echo "::add-path::/opt/ghc/8.6.5/bin"
           echo "::add-path::/opt/cabal/3.4/bin"
           echo "::add-path::$HOME/.cabal/bin"
-      - name: Update Hackage index
-        run: cabal v2-update
       - uses: actions/checkout@v2
       - name: Package project
-        run: python3 release.py
+        run: |
+          cabal v2-sdist --builddir=sdist Cabal cabal-install
+          python3 release.py --tarlib=sdist/sdist/Cabal-*.tar.gz --tarexe=sdist/sdist/cabal-install-*.tar.gz
       - uses: actions/upload-artifact@v2
         with:
           name: cabal-macos
@@ -91,12 +91,14 @@ jobs:
           ghc --version
           cabal --version
           cabal user-config init -a "http-transport: plain-http" -a "store-dir: C:\SR" -f -v3
-      - name: Update Hackage index
-        run: cabal v2-update
       - uses: actions/checkout@v2
       - name: Package project
-        run: python release.py
+        # We hardcode version 3 here to make first glob not overlap with cabal-install
+        run: |
+          cabal v2-sdist --builddir=sdist Cabal cabal-install
+          New-Item -Path "_build" -ItemType "directory"
+          python release.py --tarlib=$(Resolve-Path -Path sdist/sdist/Cabal-3.*.tar.gz) --tarexe=$(Resolve-Path -Path sdist/sdist/cabal-install-3.*.tar.gz)
       - uses: actions/upload-artifact@v2
         with:
-          name: cabal-macos
+          name: cabal-windows
           path: _build/artifacts/*

release.py

@@ -35,10 +35,11 @@
     ('compiler', Path),
     ('cabal', Path),
     ('indexstate', str),
-    ('rootdir', Path),
     ('builddir', Path),
     ('static', bool),
     ('ofdlocking', bool),
+    ('tarlib', Path),
+    ('tarexe', Path),
 ])
 
 # utils
@@ -110,7 +111,6 @@ def step_makedirs(args: Args):
     (args.builddir / 'bin').mkdir(parents=True, exist_ok=True)
     (args.builddir / 'cabal').mkdir(parents=True, exist_ok=True)
 
-# 57936384
 def step_config(args: Args):
     splitsections = ''
     if platform.system() == 'Linux':
@@ -123,6 +123,7 @@ def step_config(args: Args):
         if msysbin.is_dir():
             extraprogpath = extraprogpath + "," + str(msysbin)
 
+    # cabal.config
     config = dedent(f"""
         repository hackage.haskell.org
           url: http://hackage.haskell.org/
@@ -154,20 +155,34 @@ def step_config(args: Args):
     with open(args.builddir / 'cabal' / 'config', 'w') as f:
         f.write(config)
 
-    cabal_project_local =''
+    # cabal.project
+    cabal_project = dedent(f"""
+        packages: {args.tarlib}
+        packages: {args.tarexe}
+        tests: False
+        benchmarks: False
+        optimization: True
+
+        package Cabal
+          ghc-options: -fexpose-all-unfoldings -fspecialise-aggressively
+
+        package parsec
+          ghc-options: -fexpose-all-unfoldings
+    """)
+
     if args.static:
         # --enable-executable-static doesn't affect "non local" executables, as in v2-install project
-        cabal_project_local += dedent("""
+        cabal_project += dedent("""
             package cabal-install
                 executable-static: True
         """)
-    cabal_project_local += dedent(f"""
+    cabal_project += dedent(f"""
         package lukko
             flags: {'+' if args.ofdlocking else '-'}ofd_locking
     """)
 
-    with open(args.rootdir / 'cabal.project.release.local', 'w') as f:
-        f.write(cabal_project_local)
+    with open(args.builddir / 'cabal.project', 'w') as f:
+        f.write(cabal_project)
 
 def make_env(args: Args):
     env = {
@@ -197,7 +212,7 @@ def step_cabal_update(args: Args):
         'v2-update',
         '-v',
         f'--index-state={args.indexstate}',
-    ], check=True, env=env)
+    ], cwd=args.builddir, check=True, env=env)
 
 def step_cabal_install(args: Args):
     env = make_env(args)
@@ -206,9 +221,9 @@ def step_cabal_install(args: Args):
         'v2-install',
         '-v',
         'cabal-install:exe:cabal',
-        '--project-file=cabal.project.release',
+        '--project-file=cabal.project',
         f'--with-compiler={args.compiler}',
-    ], check=True, env=env)
+    ], cwd=args.builddir, check=True, env=env)
 
 def step_make_archive(args: Args):
     import tempfile
@@ -267,18 +282,21 @@ def __call__(self, parser, namespace, values, option_string=None):
     parser.add_argument('-i', '--index-state', type=str, default=DEFAULT_INDEXSTATE, help='index state of Hackage to use')
     parser.add_argument('--enable-static-executable', '--disable-static-executable', dest='static', nargs=0, default=False, action=EnableDisable, help='Statically link cabal executable')
     parser.add_argument('--enable-ofd-locking', '--disable-ofd-locking', dest='ofd_locking', nargs=0, default=True, action=EnableDisable, help='OFD locking (lukko)')
+    parser.add_argument('--tarlib', dest='tarlib', required=True, metavar='LIBTAR', help='path to Cabal-version.tar.gz')
+    parser.add_argument('--tarexe', dest='tarexe', required=True, metavar='EXETAR', help='path to cabal-install-version.tar.gz')
+    parser.add_argument('--builddir', dest='builddir', type=str, default='_build', help='build directory')
 
     args = parser.parse_args()
 
-    rootdir = Path('.').resolve()
     args = Args(
         compiler   = Path(shutil.which(args.with_compiler)),
         cabal      = Path(shutil.which(args.with_cabal)),
         indexstate = args.index_state,
-        rootdir    = rootdir,
-        builddir   = rootdir.resolve() / '_build',
+        builddir   = Path(args.builddir).resolve(),
         static     = args.static,
-        ofdlocking = args.ofd_locking
+        ofdlocking = args.ofd_locking,
+        tarlib     = Path(args.tarlib).resolve(),
+        tarexe     = Path(args.tarexe).resolve()
     )
 
     print(dedent(f"""
@@ -288,6 +306,8 @@ def __call__(self, parser, namespace, values, option_string=None):
         builddir:    {args.builddir}
         static:      {args.static}
         ofd-locking: {args.ofdlocking}
+        lib-tarball: {args.tarlib}
+        exe-tarball: {args.tarexe}
     """))
 
     # Check tools

templates/ci-artifacts.template.yml

@@ -26,11 +26,11 @@ jobs:
           add-apt-repository ppa:deadsnakes/ppa
           apt-get update
           apt-get install -y python3.8
-      - name: Update Hackage index
-        run: cabal v2-update
       - uses: actions/checkout@v2
       - name: Package project
-        run: python3.8 release.py
+        run: |
+          cabal v2-sdist --builddir=sdist Cabal cabal-install
+          python3.8 release.py --tarlib=sdist/sdist/Cabal-*.tar.gz --tarexe=sdist/sdist/cabal-install-*.tar.gz
       - uses: actions/upload-artifact@v2
         with:
           name: cabal-linux
@@ -61,11 +61,11 @@ jobs:
           echo "::add-path::/opt/ghc/8.6.5/bin"
           echo "::add-path::/opt/cabal/3.4/bin"
           echo "::add-path::$HOME/.cabal/bin"
-      - name: Update Hackage index
-        run: cabal v2-update
       - uses: actions/checkout@v2
       - name: Package project
-        run: python3 release.py
+        run: |
+          cabal v2-sdist --builddir=sdist Cabal cabal-install
+          python3 release.py --tarlib=sdist/sdist/Cabal-*.tar.gz --tarexe=sdist/sdist/cabal-install-*.tar.gz
       - uses: actions/upload-artifact@v2
         with:
           name: cabal-macos
@@ -91,12 +91,14 @@ jobs:
           ghc --version
           cabal --version
           cabal user-config init -a "http-transport: plain-http" -a "store-dir: C:\SR" -f -v3
-      - name: Update Hackage index
-        run: cabal v2-update
       - uses: actions/checkout@v2
       - name: Package project
-        run: python release.py
+        # We hardcode version 3 here to make first glob not overlap with cabal-install
+        run: |
+          cabal v2-sdist --builddir=sdist Cabal cabal-install
+          New-Item -Path "_build" -ItemType "directory"
+          python release.py --tarlib=$(Resolve-Path -Path sdist/sdist/Cabal-3.*.tar.gz) --tarexe=$(Resolve-Path -Path sdist/sdist/cabal-install-3.*.tar.gz)
       - uses: actions/upload-artifact@v2
         with:
-          name: cabal-macos
+          name: cabal-windows
           path: _build/artifacts/*