diff --git a/docs/user-reference.rst b/docs/user-reference.rst index 9103212..aed4b97 100644 --- a/docs/user-reference.rst +++ b/docs/user-reference.rst @@ -304,6 +304,30 @@ list of supported options: as this is a main `repo` use case. +http fetcher +^^^^^^^^^^^^^^ + +`http` fetcher is used to download a file via HTTP or HTTPS protocol. It uses +:code:`curl` tool to do so. Full list of supported options: + +.. code-block:: yaml + + type: http # Selects `http` fetcher + url: "https://example.com/file.txt" + filename: "file.txt" + dir: "." + +* :code:`type` - mandatory - should be :code:`http` to use `http` + fetcher. Use the same type even if you are downloading over HTTPS + protocol. +* :code:`url` - mandatory - URL of a file do be downloaded +* :code:`filename` - optional (in most cases) - name of the output + file. If omitted, `moulin` will try to guess it from a URL. But if + can't do so, it will ask you to provide filename manually. +* :code:`dir` - optional - directory name where store a downloaded + file. If it is omitted, `moulin` will use :code:`"."` to download a + file right into the component's root directory. + unpack fetcher ^^^^^^^^^^^^^^ diff --git a/moulin/fetchers/http.py b/moulin/fetchers/http.py new file mode 100644 index 0000000..6c0da6f --- /dev/null +++ b/moulin/fetchers/http.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2024 EPAM Systems +"""HTTP(S) fetcher module""" + +import os.path +from typing import List +from moulin.yaml_helpers import YAMLProcessingError +from moulin.yaml_wrapper import YamlValue +from moulin import ninja_syntax + + +def get_fetcher(conf: YamlValue, build_dir: str, generator: ninja_syntax.Writer): + """Construct and return HTTP/S fetcher object""" + return HTTPFetcher(conf, build_dir, generator) + + +def gen_build_rules(generator: ninja_syntax.Writer): + """Generate build rules using Ninja generator""" + generator.rule("curl_download", command="curl $url -o $out ", description="curl download $url") + generator.newline() + + +def _guess_filename(url: str): + # TODO: Add support for corner cases + if url.endswith("/"): + return None + return url.split("/")[-1] + + +class HTTPFetcher: + """HTTP/S fetcher class. Provides methods to generate rules for downloading files over HTTP/S""" + + def __init__(self, conf: YamlValue, build_dir: str, generator: ninja_syntax.Writer): + self.conf = conf + self.build_dir = build_dir + self.generator = generator + self.url = conf["url"].as_str + dirname = conf.get("dir", default=".").as_str + self.download_dir = os.path.join(build_dir, dirname) + filename = conf.get("filename", default="").as_str + if not filename: + filename = _guess_filename(self.url) + if not filename: + raise YAMLProcessingError("Can't determine output file name for HTTP/S download", + conf.mark) + self.output_file = os.path.join(self.download_dir, filename) + + def gen_fetch(self): + """Generate instruction to download a file""" + + self.generator.build(self.output_file, "curl_download", variables={"url": self.url}) + self.generator.newline() + + return self.output_file + + def get_file_list(self) -> List[str]: + return [self.output_file] + + def capture_state(self): + """ + Capture state, but it is not applicable for this fetcher. Do nothing. + """ + pass