Merge pull request #1295 from conda-forge/rebased-cfep-13

ENH CFEP-13 new CI package

conda_smithy/cli.py

@@ -566,7 +566,7 @@ class GenerateFeedstockToken(Subcommand):
     def __init__(self, parser):
         super(GenerateFeedstockToken, self).__init__(
             parser,
-            "Generate a feedstock token at ~/.conda-smithy/{user or org}_{project}_feedstock.token",
+            "Generate a feedstock token at ~/.conda-smithy/{user or org}_{project}.token",
         )
         scp = self.subcommand_parser
         scp.add_argument(
@@ -594,7 +594,7 @@ def __call__(self, args):
 
         generate_and_write_feedstock_token(owner, repo)
         print(
-            "Your feedstock token has been generated at ~/.conda-smithy/%s_%s_feedstock.token\n"
+            "Your feedstock token has been generated at ~/.conda-smithy/%s_%s.token\n"
             "This token is stored in plaintext so be careful!" % (owner, repo)
         )
 

conda_smithy/configure_feedstock.py

@@ -699,6 +699,22 @@ def _render_ci_provider(
         else:
             forge_config["upload_script"] = "upload_or_check_non_existence"
 
+        # if the recipe has its own conda_forge_ci_setup package, then
+        # install that
+        if os.path.exists(
+            os.path.join(
+                forge_dir,
+                forge_config["recipe_dir"],
+                "conda_forge_ci_setup",
+                "__init__.py",
+            )
+        ) and os.path.exists(
+            os.path.join(forge_dir, forge_config["recipe_dir"], "setup.py",)
+        ):
+            forge_config["local_ci_setup"] = True
+        else:
+            forge_config["local_ci_setup"] = False
+
         # hook for extending with whatever platform specific junk we need.
         #     Function passed in as argument
         for platform, enable in zip(platforms, enable_platform):
@@ -1382,7 +1398,8 @@ def _load_forge_config(forge_dir, exclusive_config_file):
         },
         "recipe_dir": "recipe",
         "skip_render": [],
-        "bot": {"automerge": False,},
+        "bot": {"automerge": False},
+        "conda_forge_output_validation": False,
     }
 
     forge_yml = os.path.join(forge_dir, "conda-forge.yml")
@@ -1731,7 +1748,6 @@ def main(
     check_version_uptodate(r, "conda-smithy", __version__, error_on_warn)
 
     forge_dir = os.path.abspath(forge_file_directory)
-
     if exclusive_config_file is not None:
         exclusive_config_file = os.path.join(forge_dir, exclusive_config_file)
         if not os.path.exists(exclusive_config_file):
@@ -1743,6 +1759,7 @@ def main(
         )
 
     config = _load_forge_config(forge_dir, exclusive_config_file)
+    config["feedstock_name"] = os.path.basename(forge_dir)
 
     for each_ci in ["travis", "circle", "appveyor", "drone"]:
         if config[each_ci].pop("enabled", None):

conda_smithy/feedstock_tokens.py

@@ -12,7 +12,7 @@
 The `generate-feedstock-token` command must be called before the `register-feedstock-token`
 command. It generates a random token and writes it to
 
-    ~/.conda-smithy/{user or org}_{repo w/o '-feedstock'}_feedstock.token
+    ~/.conda-smithy/{user or org}_{repo}.token
 
 Then when you call `register-feedstock-token`, the generated token is placed
 as a secret variable on the CI services. It is also hashed using `scrypt` and
@@ -32,17 +32,10 @@
 import scrypt
 
 
-def _munge_project(project):
-    if project.endswith("-feedstock"):
-        return project[: -len("-feedstock")]
-    else:
-        return project
-
-
 def generate_and_write_feedstock_token(user, project):
     """Generate a feedstock token and write it to
 
-        ~/.conda-smithy/{user or org}_{repo w/o '-feedstock'}_feedstock.token
+        ~/.conda-smithy/{user or org}_{repo}_feedstock.token
 
     This function will fail if the token file already exists.
     """
@@ -54,9 +47,7 @@ def generate_and_write_feedstock_token(user, project):
         try:
             token = secrets.token_hex(32)
             pth = os.path.join(
-                "~",
-                ".conda-smithy",
-                "%s_%s_feedstock.token" % (user, _munge_project(project)),
+                "~", ".conda-smithy", "%s_%s.token" % (user, project),
             )
             pth = os.path.expanduser(pth)
             if os.path.exists(pth):
@@ -95,7 +86,7 @@ def generate_and_write_feedstock_token(user, project):
 def read_feedstock_token(user, project):
     """Read the feedstock token from
 
-        ~/.conda-smithy/{user or org}_{repo w/o '-feedstock'}_feedstock.token
+        ~/.conda-smithy/{user or org}_{repo}.token
 
     In order to not spill any tokens to stdout/stderr, this function
     should be used in a `try...except` block with stdout/stderr redirected
@@ -106,25 +97,22 @@ def read_feedstock_token(user, project):
 
     # read the token
     user_token_pth = os.path.join(
-        "~",
-        ".conda-smithy",
-        "%s_%s_feedstock.token" % (user, _munge_project(project)),
+        "~", ".conda-smithy", "%s_%s.token" % (user, project),
     )
     user_token_pth = os.path.expanduser(user_token_pth)
 
     if not os.path.exists(user_token_pth):
-        err_msg = (
-            "No token found in '~/.conda-smithy/%s_%s_feedstock.token'"
-            % (user, _munge_project(project))
+        err_msg = "No token found in '~/.conda-smithy/%s_%s.token'" % (
+            user,
+            project,
         )
     else:
         with open(user_token_pth, "r") as fp:
             feedstock_token = fp.read().strip()
         if not feedstock_token:
             err_msg = (
-                "Empty token found in '~/.conda-smithy/"
-                "%s_%s_feedstock.token'"
-            ) % (user, _munge_project(project))
+                "Empty token found in '~/.conda-smithy/" "%s_%s.token'"
+            ) % (user, project)
             feedstock_token = None
     return feedstock_token, err_msg
 
@@ -158,9 +146,7 @@ def feedstock_token_exists(user, project, token_repo):
                 .replace("${GH_TOKEN}", github_token)
             )
             git.Repo.clone_from(_token_repo, tmpdir, depth=1)
-            token_file = os.path.join(
-                tmpdir, "tokens", _munge_project(project) + ".json",
-            )
+            token_file = os.path.join(tmpdir, "tokens", project + ".json",)
 
             if os.path.exists(token_file):
                 exists = True
@@ -217,9 +203,7 @@ def is_valid_feedstock_token(user, project, feedstock_token, token_repo):
                 .replace("${GH_TOKEN}", github_token)
             )
             git.Repo.clone_from(_token_repo, tmpdir, depth=1)
-            token_file = os.path.join(
-                tmpdir, "tokens", _munge_project(project) + ".json",
-            )
+            token_file = os.path.join(tmpdir, "tokens", project + ".json",)
 
             if os.path.exists(token_file):
                 with open(token_file, "r") as fp:
@@ -294,9 +278,7 @@ def register_feedstock_token(user, project, token_repo):
                 .replace("${GH_TOKEN}", github_token)
             )
             repo = git.Repo.clone_from(_token_repo, tmpdir, depth=1)
-            token_file = os.path.join(
-                tmpdir, "tokens", _munge_project(project) + ".json",
-            )
+            token_file = os.path.join(tmpdir, "tokens", project + ".json",)
 
             # don't overwrite existing tokens
             # check again since there might be a race condition

conda_smithy/templates/appveyor.yml.tmpl

@@ -47,7 +47,11 @@ install:
 
     # Configure the VM.
     # Tell conda we want an updated version of conda-forge-ci-setup and conda-build
-    - cmd: conda.exe install -n root -c conda-forge --quiet --yes conda-forge-ci-setup=2 conda-build
+    - cmd: conda.exe install -n root -c conda-forge --quiet --yes conda-forge-ci-setup=3 conda-build pip
+    {%- if local_ci_setup %}
+    - cmd: conda.exe uninstall --quiet --yes --force conda-forge-ci-setup
+    - cmd: pip install --no-deps .\{{ recipe_dir }}\.
+    {%- endif %}
     - cmd: setup_conda_rc .\ .\{{ recipe_dir }} .\.ci_support\%CONFIG%.yaml
     {% if build_setup -%}
     {{ build_setup }}{%- endif %}
@@ -56,10 +60,13 @@ install:
 build: off
 
 test_script:
-    - conda.exe build {{ recipe_dir }} -m .ci_support\%CONFIG%.yaml
+    - cmd: conda.exe build {{ recipe_dir }} -m .ci_support\%CONFIG%.yaml
+{%- if conda_forge_output_validation %}
+    - cmd: validate_recipe_outputs "{{ feedstock_name }}"
+{%- endif %}
 deploy_script:
     - set "GIT_BRANCH=%APPVEYOR_REPO_BRANCH%"
 {%- if upload_on_branch %}
     - set "UPLOAD_ON_BRANCH={{ upload_on_branch }}"
 {%- endif %}
-    - cmd: upload_package .\ .\{{ recipe_dir }} .ci_support\%CONFIG%.yaml
+    - cmd: upload_package {% if conda_forge_output_validation %}--validate --feedstock-name="{{ feedstock_name }}"{% endif %} .\ .\{{ recipe_dir }} .ci_support\%CONFIG%.yaml

conda_smithy/templates/azure-pipelines-linux.yml.tmpl

@@ -37,3 +37,7 @@ jobs:
     displayName: Run docker build
     env:
       BINSTAR_TOKEN: $(BINSTAR_TOKEN)
+{%- if conda_forge_output_validation %}
+      FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN)
+      STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN)
+{%- endif %}

conda_smithy/templates/azure-pipelines-osx.yml.tmpl

@@ -31,3 +31,7 @@ jobs:
     displayName: Run OSX build
     env:
       BINSTAR_TOKEN: $(BINSTAR_TOKEN)
+{%- if conda_forge_output_validation %}
+      FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN)
+      STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN)
+{%- endif %}

conda_smithy/templates/azure-pipelines-win.yml.tmpl

@@ -60,15 +60,23 @@ jobs:
 
     - task: CondaEnvironment@1
       inputs:
-        packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=2' # Optional
+        packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=3 pip' # Optional
         installOptions: "-c conda-forge"
         updateConda: true
       displayName: Install conda-build and activate environment
 
     - script: set PYTHONUNBUFFERED=1
+      displayName: Set PYTHONUNBUFFERED
 
     # Configure the VM
-    - script: setup_conda_rc .\ ".\{{ recipe_dir }}" .\.ci_support\%CONFIG%.yaml
+    - script: |
+        call activate base
+        {%- if local_ci_setup %}
+        conda.exe uninstall --quiet --yes --force conda-forge-ci-setup
+        pip install --no-deps ".\{{ recipe_dir }}\."
+        {%- endif %}
+        setup_conda_rc .\ ".\{{ recipe_dir }}" .\.ci_support\%CONFIG%.yaml
+      displayName: conda-forge CI setup
 
     {% if build_setup -%}
     # Configure the VM.
@@ -97,14 +105,25 @@ jobs:
         PYTHONUNBUFFERED: 1
       condition: not(contains(variables['CONFIG'], 'vs2008'))
 
+{%- if conda_forge_output_validation %}
+    - script: |
+        call activate base
+        validate_recipe_outputs "{{ feedstock_name }}"
+      displayName: Validate Recipe Outputs
+{%- endif %}
+
     - script: |
         set "GIT_BRANCH=%BUILD_SOURCEBRANCHNAME%"
 {%- if upload_on_branch %}
         set "UPLOAD_ON_BRANCH={{ upload_on_branch }}"
 {%- endif %}
         call activate base
-        upload_package .\ ".\{{ recipe_dir }}" .ci_support\%CONFIG%.yaml
+        upload_package {% if conda_forge_output_validation %}--validate --feedstock-name="{{ feedstock_name }}"{% endif %} .\ ".\{{ recipe_dir }}" .ci_support\%CONFIG%.yaml
       displayName: Upload package
       env:
         BINSTAR_TOKEN: $(BINSTAR_TOKEN)
+{%- if conda_forge_output_validation %}
+        FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN)
+        STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN)
+{%- endif %}
       condition: and(succeeded(), not(eq(variables['UPLOAD_PACKAGES'], 'False')))

conda_smithy/templates/build_steps.sh.tmpl

@@ -19,7 +19,11 @@ conda-build:
 
 CONDARC
 
-conda install --yes --quiet conda-forge-ci-setup=2 conda-build -c conda-forge
+conda install --yes --quiet conda-forge-ci-setup=3 conda-build pip -c conda-forge
+{% if local_ci_setup %}
+conda uninstall --quiet --yes --force conda-forge-ci-setup
+pip install --no-deps ${RECIPE_ROOT}/.
+{%- endif %}
 
 # set up the condarc
 setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
@@ -35,8 +39,12 @@ make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
 conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \
     --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml"
 
+{%- if conda_forge_output_validation %}
+validate_recipe_outputs "{{ feedstock_name }}"
+{%- endif %}
+
 if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then
-    upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
+    upload_package {% if conda_forge_output_validation %}--validate --feedstock-name="{{ feedstock_name }}"{% endif %} "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
 fi
 
 touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}"

conda_smithy/templates/drone.yml.tmpl

@@ -15,8 +15,15 @@ steps:
     CONFIG: {{ data.config_name }}
     UPLOAD_PACKAGES: {{ data.upload }}
     PLATFORM: {{ data.platform }}
-    BINSTAR_TOKEN: 
+    BINSTAR_TOKEN:
       from_secret: BINSTAR_TOKEN
+{%- if conda_forge_output_validation %}
+    FEEDSTOCK_TOKEN:
+      from_secret: FEEDSTOCK_TOKEN
+    STAGING_BINSTAR_TOKEN:
+      from_secret: STAGING_BINSTAR_TOKEN
+{%- endif %}
+
   {%- if 'linux' in data.platform %}
   commands:
     - export FEEDSTOCK_ROOT="$DRONE_WORKSPACE"

conda_smithy/templates/run_docker_build.sh.tmpl

@@ -69,6 +69,10 @@ export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}"
            -e GIT_BRANCH \
            -e UPLOAD_ON_BRANCH \
            -e CI \
+{%- if conda_forge_output_validation %}
+           -e FEEDSTOCK_TOKEN \
+           -e STAGING_BINSTAR_TOKEN \
+{%- endif %}
            $DOCKER_IMAGE \
            {{ docker.command }} \
            /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh

conda_smithy/templates/run_osx_build.sh.tmpl

@@ -22,8 +22,13 @@ fi
 source ${HOME}/miniforge3/etc/profile.d/conda.sh
 conda activate base
 
-echo -e "\n\nInstalling conda-forge-ci-setup=2 and conda-build."
-conda install -n base --quiet --yes conda-forge-ci-setup=2 conda-build
+echo -e "\n\nInstalling conda-forge-ci-setup=3 and conda-build."
+conda install -n base --quiet --yes conda-forge-ci-setup=3 conda-build pip
+
+{% if local_ci_setup %}
+conda uninstall --quiet --yes --force conda-forge-ci-setup
+pip install --no-deps {{ recipe_dir }}/.
+{%- endif %}
 
 echo -e "\n\nSetting up the condarc and mangling the compiler."
 setup_conda_rc ./ ./{{ recipe_dir }} ./.ci_support/${CONFIG}.yaml
@@ -47,7 +52,11 @@ echo -e "\n\nMaking the build clobber file and running the build."
 make_build_number ./ ./{{ recipe_dir }} ./.ci_support/${CONFIG}.yaml
 conda build ./{{ recipe_dir }} -m ./.ci_support/${CONFIG}.yaml --clobber-file ./.ci_support/clobber_${CONFIG}.yaml
 
+{%- if conda_forge_output_validation %}
+validate_recipe_outputs "{{ feedstock_name }}"
+{%- endif %}
+
 if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then
   echo -e "\n\nUploading the packages."
-  upload_package ./ ./{{ recipe_dir }} ./.ci_support/${CONFIG}.yaml
+  upload_package {% if conda_forge_output_validation %}--validate --feedstock-name="{{ feedstock_name }}"{% endif %} ./ ./{{ recipe_dir }} ./.ci_support/${CONFIG}.yaml
 fi

news/token_validation.rst

@@ -0,0 +1,30 @@
+**Added:**
+
+* Added new `pip-install`-based hooks for using a local copy of the
+  `conda-forge-ci-setup` package.
+
+**Changed:**
+
+* Renamed the feedstock token output files to not munge "-feedstock" from
+  the names.
+
+* Bumped the default version of the `conda-forge-ci-setup` package to 3 to
+  support the new output validation service.
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* Fixed bugs in tests for feedstock tokens.
+
+**Security:**
+
+* Added code to call the feedstock output validation service. You must have
+  `conda_forge_output_validation` set to true in the `conda-forge.yml` to use
+  this feature.