ml4qgis_algorithm.py

# -*- coding: utf-8 -*-

"""
/***************************************************************************
 Ml4Qgis
                                 A QGIS plugin
 Machine learning in QGIS.
 Generated by Plugin Builder: http://g-sherman.github.io/Qgis-Plugin-Builder/
                              -------------------
        begin                : 2023-05-06
        copyright            : (C) 2023 by Mikhail Moskovchenko
        email                : moskovchenkomike@gmail.com
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""

__author__ = 'Mikhail Moskovchenko'
__date__ = '2023-05-06'
__copyright__ = '(C) 2023 by Mikhail Moskovchenko'

# This will get replaced with a git SHA1 when you do a git archive

__revision__ = '$Format:%H$'
import shutil
import os

from qgis.PyQt.QtCore import QCoreApplication
from qgis.core import (QgsProcessing,
                       QgsFeatureSink,
                       QgsProcessingAlgorithm,
                       QgsProcessingParameterFeatureSource,
                       QgsProcessingParameterFeatureSink,
                       QgsProcessingParameterVectorLayer,
                       QgsProcessingParameterRasterLayer,
                       QgsProcessingParameterFile,
                       QgsProcessingParameterFolderDestination,
                       QgsProcessingParameterCrs,
                       QgsProcessingParameterString,
                       QgsProcessingParameterBoolean,
                       QgsProcessingParameterEnum,
                       QgsProcessingParameterNumber,
                       QgsProcessingParameterMultipleLayers,
                       QgsProcessingParameterFileDestination)
                       
from .setup import check, docker_install, conda_install, python_install, getTempdir                       
from .preprocessing import sentinel2, landsat, mosaic, normalize
from .segmentation import seg_generate_tiles, seg_train, seg_test, seg_map


class DockerSetupAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        self.addParameter(
            QgsProcessingParameterString("-", "Don't mean anything at all", defaultValue="0")
        )
        

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        docker_install(feedback)
        
        return {}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Install or repair Docker container'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Setup'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
        
    def shortHelpString(self):
        return 'ML4QGIS can use Docker backend for processing.\n \
        That tool sets up the Docker image that contain the Python environment configured to do \
        machine learning tasks. \n To use Docker backend you need to: \n \
        1) Download Docker installer from <a href="https://www.docker.com">Docker official site</a>. \n \
        2. Install Docker. \n 3. Run Docker. \n 4. Run this tool. \n \
        Please always make sure that Docker is running every time you use any ML4QGIS script with Docker \
        backend. \n We also recommend to use swap file because ML4QGIS scripts usually use lot of RAM.'

    def createInstance(self):
        return DockerSetupAlgorithm()
        
        
class CondaSetupAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterFile(
                'CONDAPATH',
                self.tr('Conda folder'),
                behavior = 1
            )
        )
        self.addParameter(
            QgsProcessingParameterString(
                "ENV", 
                self.tr("Environment name"),
                optional = True,
                defaultValue = 'rsp'
            )
        )
        

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        path = self.parameterAsFile(parameters, 'CONDAPATH', context)
        env = self.parameterAsString(parameters, 'ENV', context)
        conda_install(path, env, feedback)
        
        return {}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Setup Conda access config'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Setup'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
        
    def shortHelpString(self):
        path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'environment.yml')
        return 'ML4QGIS can use Conda backend for processing.\n \
        That tool sets up a config that will be used to access Conda env with installed requirements \
        to run ML4QGIS scripts. You need to install Conda and set up the environment manually. \n \
        1. Install <a href="https://www.anaconda.com/">Anaconda</a> or \
        <a href="https://docs.conda.io/en/latest/miniconda.html">Miniconda</a>. \n \
        2. Run a command "conda env create -f ' + path + '" or manually create Conda env that have \
        <a href="https://remote-sensing-processor.readthedocs.io/en/latest/install.html">remote_sensing_processor</a> \
        and all its dependencies installed. \n \
        3. Select Conda folder destination. By default it is C:/Users/username/Anaconda3 or in \
        C:/ProgramData/Anaconda3 on Windows, /home/username/anaconda/ on Linux and /Users/username/anaconda on Mac \
        If you have installed miniconda, the folder will be named miniconda instead of anaconda. \n \
        4. Define environment name. By default it is "rsp". \n 5. Run this tool.\n \
        We also recommend to use swap file because ML4QGIS scripts usually use lot of RAM.'

    def createInstance(self):
        return CondaSetupAlgorithm()
        
        
class PythonSetupAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterFile(
                'PYTHONPATH',
                self.tr('Python venv folder'),
                behavior = 1
            )
        )
        
    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        path = self.parameterAsFile(parameters, 'PYTHONPATH', context)
        python_install(path, feedback)
        
        return {}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Setup Python access config'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Setup'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
        
    def shortHelpString(self):
        return 'ML4QGIS can use Python venv as a backend for processing.\n \
        You need to set up the Python virtual environment configured to do machine learning tasks. \n \
        1. Download and install <a href="https://www.python.org/">Python</a>. \n \
        2. Create and activate new <a href="https://docs.python.org/3/library/venv.html">Virtual environment</a>. \n \
        3. Install <a href="https://remote-sensing-processor.readthedocs.io/en/latest/install.html">remote_sensing_processor</a> \
        and all its dependencies in a newly created venv. \n \
        4. Select path to your venv. \n 5. Run this tool.\n \
        We also recommend to use swap file because ML4QGIS scripts usually use lot of RAM. '

    def createInstance(self):
        return PythonSetupAlgorithm()
        
        
class TempDirAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        self.addParameter(
            QgsProcessingParameterFolderDestination(
                "TEMP",
                self.tr('Temporary directory'),
                defaultValue = getTempdir()
            )
        )
        

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        outputDest = self.parameterAsString(parameters, "TEMP", context)
        tempconfig = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tempdir.json')
        with open(tempconfig, 'r') as f:
            data = json.load(f)
        data["tempdir"] = outputDest
        with open(tempconfig, 'w') as f:
            json.dump(data, f)
        
        return {}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Set temp directory'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Setup'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def shortHelpString(self):
        return 'ML4QGIS writes all used data to temporary directory to protect the original files from being damaged. \n \
        The drive or volume where temporary directory is located must have enough free space.'

    def createInstance(self):
        return TempDirAlgorithm()


class Sentinel2Algorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    # Constants used to refer to parameters and outputs. They will be
    # used when calling the algorithm from another algorithm, or when
    # calling from the QGIS console.

    OUTPUT = 'OUTPUT'
    INPUT = 'INPUT'

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "BACKEND",
                "Backend",
                options=['Python venv', 'Conda', 'Docker container'],
                defaultValue = 0
            )
        )
        # We add the input vector features source. It can have any kind of
        # geometry.
        self.addParameter(
            QgsProcessingParameterFile(
                self.INPUT,
                self.tr('Input archive'),
                extension = 'zip'
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "SEN2COR",
                "Sen2cor atmospheric correction",
                defaultValue = True,
                optional = True
            )
        )
                                                        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "SUPERRES",
                "Super Resolution (uses GPU)",
                defaultValue = True, 
                optional = True
            )
        )
                                                        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "CLOUDMASK",
                "Mask clouds",
                defaultValue = True, 
                optional = True
            )
        )
                                                        
        self.addParameter(
            QgsProcessingParameterCrs(
                "CRS",
                "CRS",
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterVectorLayer(
                'CLIP',
                'Vector layer to clip by',
                [QgsProcessing.TypeVectorPolygon],
                optional = True
            )
        )
        
        
        # We add a feature sink in which to store our processed features (this
        # usually takes the form of a newly created vector layer when the
        # algorithm is run in QGIS).
        self.addParameter(
            QgsProcessingParameterFolderDestination(
                self.OUTPUT,
                self.tr('Output folder')
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        backend = self.parameterAsEnum(parameters, "BACKEND", context)
        backend = ['python', 'conda', 'docker'][backend]
        inputLayer = self.parameterAsFile(parameters, self.INPUT, context)
        inputSen2Cor = self.parameterAsBool(parameters, "SEN2COR", context)
        inputSuperRes = self.parameterAsBool(parameters, "SUPERRES", context)
        inputCloudMask = self.parameterAsBool(parameters, "CLOUDMASK", context)
        #inputClip = self.parameterAsSource(parameters, "CLIP", context)
        inputClip = self.parameterAsVectorLayer(parameters, "CLIP", context)
        outputCrs = self.parameterAsCrs(parameters, "CRS", context)
        outputDest = self.parameterAsString(parameters, self.OUTPUT, context)
        sentinel2(backend, inputLayer, inputSen2Cor, inputSuperRes, inputCloudMask, inputClip, outputCrs, outputDest, feedback)
        return {self.OUTPUT: outputDest}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Sentinel-2 preprocessing'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Preprocessing'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def shortHelpString(self):
        return 'Preprocess Sentinel-2 imagery. \n \
        Backend: select a backend to run script. \n \
        Input archive: a path to Sentinel-2 zip archive. \n \
        Sen2Cor atmospheric correction: To perform a sen2cor atmospheric correction you need to have Sen2Cor installed. \
        Required version is 2.11 for Windows and Linux and 2.9 for Mac OS. Here is an \
        <a href="http://wiki.awf.forst.uni-goettingen.de/wiki/index.php/Installation_of_SNAP">instruction</a> \
        how to install it. \n \
        Super Resolution: Is upscaling 20- and 60-m bands to 10 m resolution needed. \
        May run very slow if GPU does not support CUDA. \n \
        Mask clouds: removes clouds from image. \n \
        CRS: you can define projection in which output data should be. \n \
        Vector layer to clip by: path to vector file to be used to crop the image. \n \
        Output folder: directory where preprocessed image will be saved.'

    def createInstance(self):
        return Sentinel2Algorithm()


class LandsatAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    # Constants used to refer to parameters and outputs. They will be
    # used when calling the algorithm from another algorithm, or when
    # calling from the QGIS console.

    OUTPUT = 'OUTPUT'
    INPUT = 'INPUT'

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "BACKEND",
                "Backend",
                options=['Python venv', 'Conda', 'Docker container'],
                defaultValue = 0
            )
        )
        # We add the input vector features source. It can have any kind of
        # geometry.
        self.addParameter(
            QgsProcessingParameterFile(
                self.INPUT,
                self.tr('Input archive'),
                fileFilter = 'Landsat archives(*.tar *.tar.gz)'
            )
        )
                                                        
        self.addParameter(
            QgsProcessingParameterCrs(
                "CRS",
                "CRS",
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "CLOUDMASK",
                "Mask clouds",
                defaultValue = True, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "PANSHARPEN",
                "Pansharpen",
                defaultValue = True, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "KEEP_PAN",
                "Keep pansharpening band",
                defaultValue = False, 
                optional = True
            )
        )
                                                        
        self.addParameter(
            QgsProcessingParameterEnum(
                "RESAMPLING",
                "Resampling algorithm",
                options=['bilinear', 'cubic', 'cubic_spline', 'lanczos', 'average', 'mode', 'max', 'min', 'med', 'q1', 'q3', 'sum', 'rms', 'nearest'],
                defaultValue = 0, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "TEMPERATURE",
                "Temperature units",
                options=['k', 'c'],
                defaultValue = 0, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterVectorLayer(
                'CLIP',
                'Vector layer to clip by',
                [QgsProcessing.TypeVectorPolygon],
                optional = True
            )
        )
        
        
        # We add a feature sink in which to store our processed features (this
        # usually takes the form of a newly created vector layer when the
        # algorithm is run in QGIS).
        self.addParameter(
            QgsProcessingParameterFolderDestination(
                self.OUTPUT,
                self.tr('Output folder')
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        backend = self.parameterAsEnum(parameters, "BACKEND", context)
        backend = ['python', 'conda', 'docker'][backend]
        inputLayer = self.parameterAsFile(parameters, self.INPUT, context)
        inputCloudMask = self.parameterAsBool(parameters, "CLOUDMASK", context)
        inputPansharpen = self.parameterAsBool(parameters, "PANSHARPEN", context)
        inputKeepPan = self.parameterAsBool(parameters, "KEEP_PAN", context)
        inputResampling = self.parameterAsEnum(parameters, "RESAMPLING", context)
        inputResampling = ['bilinear', 'cubic', 'cubic_spline', 'lanczos', 'average', 'mode', 'max', 'min', 'med', 'q1', 'q3', 'sum', 'rms', 'nearest'][inputResampling]
        inputTemperature = self.parameterAsEnum(parameters, "TEMPERATURE", context)
        inputTemperature = ['k', 'c'][inputTemperature]
        #inputClip = self.parameterAsSource(parameters, "CLIP", context)
        inputClip = self.parameterAsVectorLayer(parameters, "CLIP", context)
        outputCrs = self.parameterAsCrs(parameters, "CRS", context)
        outputDest = self.parameterAsString(parameters, self.OUTPUT, context)
        landsat(backend, inputLayer, inputCloudMask, inputPansharpen, inputKeepPan, inputResampling, inputTemperature, inputClip, outputCrs, outputDest, feedback)
        return {self.OUTPUT: outputDest}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Landsat preprocessing'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Preprocessing'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def shortHelpString(self):
        return 'Preprocess Landsat imagery. \n \
        Backend: select a backend to run script. \n \
        Input archive: a path to Landsat tar or tar.gz archive. \n \
        CRS: you can define projection in which output data should be. \n \
        Mask clouds: removes clouds from image. \n \
        Pansharpen: is pansharpening needed. Brovey transform is used for pansharpening Landsat 7, 8 and 9. \n \
        Keep pansharpening band: keep pansharpening band or delete it. Pansharpening band have the same \
        wavelengths as optical bands, so it does not contain any additional information to other bands.\
        Affects only Landsat 7, 8 and 9. \n \
        Resampling algorithm: resampling method that will be used to upscale bands that cannot be \
        upscaled in pansharpening operation. Affects only Landsat 7, 8 and 9. \n \
        Temperature units: convert thermal band to kelvins or celsius (no farenheit lol). \n \
        Vector layer to clip by: path to vector file to be used to crop the image. \n \
        Output folder: directory where preprocessed image will be saved.'
    
    def createInstance(self):
        return LandsatAlgorithm()


class MosaicAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    # Constants used to refer to parameters and outputs. They will be
    # used when calling the algorithm from another algorithm, or when
    # calling from the QGIS console.

    OUTPUT = 'OUTPUT'
    INPUT = 'INPUT'

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "BACKEND",
                "Backend",
                options=['Python venv', 'Conda', 'Docker container'],
                defaultValue = 0
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "MB",
                "Multi-band rasters (Sentinel-2, Landsat etc.)",
                defaultValue = False
            )
        )
        
        # We add the input vector features source. It can have any kind of
        # geometry.
        self.addParameter(
            QgsProcessingParameterMultipleLayers(
                self.INPUT,
                self.tr("Input rasters"),
                QgsProcessing.TypeFile
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "FILL",
                "Fill nodata",
                defaultValue = False, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "DISTANCE",
                "Fill distance",
                defaultValue = 250, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterVectorLayer(
                'CLIP',
                'Vector layer to clip by',
                [QgsProcessing.TypeVectorPolygon],
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterCrs(
                "CRS",
                "CRS",
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "NODATA",
                "Nodata value", 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterRasterLayer(
                "REFERENCE",
                'Reference raster',
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "RESAMPLING",
                "Resampling algorithm",
                options=['bilinear', 'cubic', 'cubic_spline', 'lanczos', 'average', 'mode', 'max', 'min', 'med', 'q1', 'q3', 'sum', 'rms', 'nearest'],
                defaultValue = 4, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "ORDER",
                "Stack raster in order from more to less nodata",
                defaultValue = False, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "KEEP",
                "Keep all Landsat channels",
                defaultValue = True, 
                optional = True
            )
        )

        # We add a feature sink in which to store our processed features (this
        # usually takes the form of a newly created vector layer when the
        # algorithm is run in QGIS).
        self.addParameter(
            QgsProcessingParameterFolderDestination(
                self.OUTPUT,
                self.tr('Output folder')
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        backend = self.parameterAsEnum(parameters, "BACKEND", context)
        backend = ['python', 'conda', 'docker'][backend]
        inputMultiBand = self.parameterAsBool(parameters, "MB", context)
        inputLayers = self.parameterAsFileList(parameters, self.INPUT, context)
        inputFill = self.parameterAsBool(parameters, "FILL", context)
        inputFillDistance = self.parameterAsInt(parameters, "DISTANCE", context)
        inputClip = self.parameterAsVectorLayer(parameters, "CLIP", context)
        outputCrs = self.parameterAsCrs(parameters, "CRS", context)
        #inputNodata = self.parameterAsInt(parameters, "NODATA", context)
        inputNodata = parameters['NODATA']
        inputReference = self.parameterAsFile(parameters, "REFERENCE", context)
        inputResampling = self.parameterAsEnum(parameters, "RESAMPLING", context)
        inputResampling = ['bilinear', 'cubic', 'cubic_spline', 'lanczos', 'average', 'mode', 'max', 'min', 'med', 'q1', 'q3', 'sum', 'rms', 'nearest'][inputResampling]
        inputNodataOrder = self.parameterAsBool(parameters, "ORDER", context)
        inputKeepAllChannels = self.parameterAsBool(parameters, "KEEP", context)
        outputDest = self.parameterAsString(parameters, self.OUTPUT, context)
        mosaic(backend, inputMultiBand, inputLayers, inputFill, inputFillDistance, inputClip, outputCrs, inputNodata, inputReference, inputResampling, inputNodataOrder, inputKeepAllChannels, outputDest, feedback)
        return {self.OUTPUT: outputDest}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Create raster mosaic'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Preprocessing'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def shortHelpString(self):
        return 'Creates mosaic from several rasters. \n \
        Backend: select a backend to run script. \n \
        Multi-band rasters: if you are merging multi-band (Sentinel-2, Landsat) or single-band (landcover maps, \
        DEMS etc.) rasters. \n \
        Input rasters: if you ate merging multi-band rasters then add one or more band from each imagery \
        product folder, else add all rasters you want to merge. \n \
        Fill nodata: is filling the gaps in the raster needed. \n \
        Fill distance: fill distance for fill nodata function. \n \
        Vector layer to clip by: path to vector file to be used to crop the image. \n \
        CRS: you can define projection in which output data should be. \n \
        Nodata value: If not set then is read from file or set to 0. \n \
        Mask clouds: removes clouds from image. \n \
        Reference raster: reference raster is needed to bring output mosaic raster to same resolution and \
        projection as other data source. Is useful when you need to use data from different sources together. \n \
        Resampling algorithm: Resampling method that will be used to reshape to a reference raster shape. \
        Use ‘nearest’ if you want to keep only class values. \n \
        Stack rasters in order from more to less data: is needed to merge images in order from images with most\
        nodata values on bottom (they usually are most distorted and cloudy) to images with less nodata on top \
        (they are usually clear). \n \
        Keep all Landsat channels: is needed only when you are merging Landsat images from different generations. \
        If checked, all bands are processed, else only bands that are present in all input images are processed \
        and others are omited. \n\
        Output folder: directory where preprocessed images will be saved.'

    def createInstance(self):
        return MosaicAlgorithm()


class NormalizeAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    # Constants used to refer to parameters and outputs. They will be
    # used when calling the algorithm from another algorithm, or when
    # calling from the QGIS console.

    OUTPUT = 'OUTPUT'
    INPUT = 'INPUT'

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "BACKEND",
                "Backend",
                options=['Python venv', 'Conda', 'Docker container'],
                defaultValue = 0
            )
        )
        # We add the input vector features source. It can have any kind of
        # geometry.
        self.addParameter(
            QgsProcessingParameterRasterLayer(
                self.INPUT,
                self.tr('Input raster')
            )
        )
                                                        
        self.addParameter(
            QgsProcessingParameterNumber(
                "MIN",
                "Min value", 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "MAX",
                "Max value", 
                optional = True
            )
        )
        
        
        # We add a feature sink in which to store our processed features (this
        # usually takes the form of a newly created vector layer when the
        # algorithm is run in QGIS).
        self.addParameter(
            QgsProcessingParameterFileDestination(
                self.OUTPUT,
                self.tr('Output raster'),
                fileFilter = '.TIF(*.tif)'   
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        backend = self.parameterAsEnum(parameters, "BACKEND", context)
        backend = ['python', 'conda', 'docker'][backend]
        inputLayer = self.parameterAsFile(parameters, self.INPUT, context)
        #inputMin = self.parameterAsDouble(parameters, "MIN", context)
        inputMin = parameters['MIN']
        #inputMax = self.parameterAsDouble(parameters, "MAX", context)
        inputMax = parameters['MAX']
        outputDest = self.parameterAsString(parameters, self.OUTPUT, context)
        normalize(backend, inputLayer, inputMin, inputMax, outputDest, feedback)
        return {self.OUTPUT: outputDest}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Normalize raster'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Preprocessing'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def shortHelpString(self):
        return 'Applies min-max normalization to input file. \n \
        Backend: select a backend to run script. \n \
        Input raster: a path to input raster\n \
        Min value: min value for normalization. If not defined then min and max of data type of input_file \
        will be used. \n \
        Max value: max value for normalization. If not defined then min and max of data type of input_file \
        will be used. \n \
        Output raster: path where normalized raster will be saved'
    
    def createInstance(self):
        return NormalizeAlgorithm()


class SegTilesAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    # Constants used to refer to parameters and outputs. They will be
    # used when calling the algorithm from another algorithm, or when
    # calling from the QGIS console.

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "BACKEND",
                "Backend",
                options=['Python venv', 'Conda', 'Docker container'],
                defaultValue = 0
            )
        )
        
        # We add the input vector features source. It can have any kind of
        # geometry.
        self.addParameter(
            QgsProcessingParameterMultipleLayers(
                "XINPUT",
                self.tr("Training data rasters (X)"),
                QgsProcessing.TypeFile
            )
        )
        
        self.addParameter(
            QgsProcessingParameterRasterLayer(
                "YINPUT",
                'Target values raster (Y)'
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "TILESIZE",
                "Tile size",
                defaultValue = 128, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "CLASSIFICATION",
                "Classification task",
                defaultValue = True, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "SHUFFLE",
                "Random shuffle tiles",
                defaultValue = False, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "TRAINSIZE",
                "Train subset size",
                defaultValue = 3, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "VALSIZE",
                "Validation subset size",
                defaultValue = 1, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "TESTSIZE",
                "Test subset size",
                defaultValue = 1, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "XNODATA",
                "X nodata value", 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "YNODATA",
                "Y nodata value", 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "XDTYPE",
                "Convert x to dtype",
                options=['', 'uint8', 'int8', 'uint16', 'int16', 'uint32', 'int32', 'float16', 'float32'],
                defaultValue = 0, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "YDTYPE",
                "Convert y to dtype",
                options=['', 'uint8', 'int8', 'uint16', 'int16', 'uint32', 'int32', 'float16', 'float32'],
                defaultValue = 0, 
                optional = True
            )
        )
        
        # We add a feature sink in which to store our processed features (this
        # usually takes the form of a newly created vector layer when the
        # algorithm is run in QGIS).
        self.addParameter(
            QgsProcessingParameterFolderDestination(
                "OUTPUT",
                self.tr('Output folder')
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        backend = self.parameterAsEnum(parameters, "BACKEND", context)
        backend = ['python', 'conda', 'docker'][backend]
        inputX = self.parameterAsFileList(parameters, "XINPUT", context)
        inputY = self.parameterAsFile(parameters, "YINPUT", context)
        inputTileSize = self.parameterAsInt(parameters, "TILESIZE", context)
        inputClassification = self.parameterAsBool(parameters, "CLASSIFICATION", context)
        inputShuffle = self.parameterAsBool(parameters, "SHUFFLE", context)
        inputTrainSize = self.parameterAsInt(parameters, "TRAINSIZE", context)
        inputValSize = self.parameterAsInt(parameters, "VALSIZE", context)
        inputTestSize = self.parameterAsInt(parameters, "TESTSIZE", context)
        #inputXNodata = self.parameterAsString(parameters, "XNODATA", context)
        inputXNodata = parameters['XNODATA']
        #inputYNodata = self.parameterAsString(parameters, "YNODATA", context)
        inputYNodata = parameters['YNODATA']
        inputXDtype = self.parameterAsEnum(parameters, "XDTYPE", context)
        inputXDtype = ['None', 'uint8', 'int8', 'uint16', 'int16', 'uint32', 'int32', 'float16', 'float32'][inputXDtype]
        inputYDtype = self.parameterAsEnum(parameters, "YDTYPE", context)
        inputYDtype = ['None', 'uint8', 'int8', 'uint16', 'int16', 'uint32', 'int32', 'float16', 'float32'][inputYDtype]
        outputDest = self.parameterAsString(parameters, "OUTPUT", context)
        seg_generate_tiles(backend, inputX, inputY, inputTileSize, inputClassification, inputShuffle, inputTrainSize, inputValSize, inputTestSize, inputXNodata, inputYNodata, inputXDtype, inputYDtype, outputDest, feedback)
        return {"OUTPUT": outputDest, "XTRAIN": str(os.path.join(outputDest, 'x_train.h5')), "YTRAIN": str(os.path.join(outputDest, 'y_train.h5')), "XVAL": str(os.path.join(outputDest, 'x_val.h5')), "YVAL": str(os.path.join(outputDest, 'y_val.h5')), "XTEST": str(os.path.join(outputDest, 'x_test.h5')), "YTEST": str(os.path.join(outputDest, 'y_test.h5')), "SAMPLES": str(os.path.join(outputDest, 'samples.pickle'))}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Generate tiles for semantic segmentation'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Semantic segmentation'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def shortHelpString(self):
        return 'Cut rasters into tiles. \n \
        Backend: select a backend to run script. \n \
        Training data rasters (X): rasters to use as training data.\n \
        Target vaules raster (Y): raster to use as target values. \n \
        Tile size: size of tiles to generate (tile_size x tile_size). \n \
        Classification task: if checked then tiles will be prepared for classification \
        (e.g. semantic segmentation) task, else will be prepared for regression task. \n \
        Random shuffle tiles: is random shuffling of samples needed. \n \
        Subset sizes: defining proportions of every subset. 3, 1, 1  with dataset size of 100 tiles will \
        generate train subset of 60 tiles, validation subset of 20 tiles and test subset of 20 tiles. \n \
        Nodata values: you can define which value in x and y rasters corresponds to nodata \
        and areas that contain nodata will be ignored while training and testing. \
        Tiles that contain only nodata in both x and y will be omited. \
        If not stated then nodata of first x file and y file will be used. \n \
        Convert to dtype: if you run out of memory, you can try to convert your data \
        to less memory consuming format.\n \
        Output folder: directory where preprocessed images will be saved.'
    
    def createInstance(self):
        return SegTilesAlgorithm()


class SegTrainAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    # Constants used to refer to parameters and outputs. They will be
    # used when calling the algorithm from another algorithm, or when
    # calling from the QGIS console.

    OUTPUT = 'OUTPUT'
    INPUT = 'INPUT'

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "BACKEND",
                "Backend",
                options=['Python venv', 'Conda', 'Docker container'],
                defaultValue = 0
            )
        )
        
        # We add the input vector features source. It can have any kind of
        # geometry.
        self.addParameter(
            QgsProcessingParameterMultipleLayers(
                "XTRAIN",
                self.tr('X train tiles *.h5 files'),
                QgsProcessing.TypeFile
            )
        )
        
        self.addParameter(
            QgsProcessingParameterMultipleLayers(
                "YTRAIN",
                self.tr('Y train tiles *.h5 files'),
                QgsProcessing.TypeFile
            )
        )
        
        self.addParameter(
            QgsProcessingParameterMultipleLayers(
                "XVAL",
                self.tr('X val tiles *.h5 files'),
                QgsProcessing.TypeFile,
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterMultipleLayers(
                "YVAL",
                self.tr('Y val tiles *.h5 files'),
                QgsProcessing.TypeFile,
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "MODEL",
                "Model",
                options=['BEiT', 'ConditionalDETR', 'Data2Vec', 'DETR', 'DPT', 'Mask2Former', 'MaskFormer', 'MobileNetV2', 'MobileViT', 'MobileViTV2', 'OneFormer with Swin backbone', 'OneFormer with ConvNeXT backbone', 'OneFormer  with ConvNeXT2 backbone', 'OneFormer  with DiNAT backbone', 'SegFormer', 'UperNet with Swin backbone', 'UperNet with ResNet backbone', 'UperNet with ConvNeXT backbone', 'UperNet with ConvNeXT2 backbone', 'DeepLabV3 with MobileNet_V3_Large backbone', 'DeepLabV3 with ResNet50 backbone', 'DeepLabV3 with ResNet101 backbone', 'FCN with ResNet50 backbone', 'FCN with ResNet101 backbone', 'LRASPP', 'Nearest Neighbors', 'Logistic Regression with lbfgs solver', 'Logistic Regression with liblinear solver', 'Logistic Regression with newton-cg solver', 'Logistic Regression with newton-cholesky solver', 'Logistic Regression with sag solver', 'Logistic Regression with saga solver', 'Ridge', 'Lasso', 'ElasticNet', 'SVM with rbf kernel', 'SVM with linear kernel', 'SVM with poly kernel', 'SVM with sigmoid kernel', 'Gaussian Process', 'Naive Bayes', 'QDA', 'Decision Tree', 'Random Forest', 'AdaBoost', 'Gradient Boosting', 'Multilayer Perceptron'],
                defaultValue = 0
            )
        )
        
        self.addParameter(
            QgsProcessingParameterFile(
                "CHECKPOINT",
                self.tr('Checkpoint of previous training'),
                fileFilter = 'Torch or sklearn models(*.ckpt *.joblib)',
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterString(
                "WEIGHTS", 
                self.tr("Pre-trained weights"),
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "EPOCHS",
                "Number of training epochs",
                defaultValue = 5, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "BATCH",
                "Batch size",
                defaultValue = 32, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "LESS",
                "Use less metrics",
                defaultValue = False, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "LR",
                "Learning rate",
                defaultValue = 1e-3, 
                optional = True,
                type=QgsProcessingParameterNumber.Double,
                maxValue = 1,
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "MULTIPROCESSING",
                "Use multiprocessing",
                defaultValue = True, 
                optional = True
            )
        )
        
        # We add a feature sink in which to store our processed features (this
        # usually takes the form of a newly created vector layer when the
        # algorithm is run in QGIS).
        self.addParameter(
            QgsProcessingParameterFileDestination(
                self.OUTPUT,
                self.tr('Output model file'),
                fileFilter = 'Torch or sklearn models(*.ckpt *.joblib)'  
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        backend = self.parameterAsEnum(parameters, "BACKEND", context)
        backend = ['python', 'conda', 'docker'][backend]
        inputXTrain = self.parameterAsFileList(parameters, "XTRAIN", context)
        inputYTrain = self.parameterAsFileList(parameters, "YTRAIN", context)
        inputXVal = self.parameterAsFileList(parameters, "XVAL", context)
        inputYVal = self.parameterAsFileList(parameters, "YVAL", context)
        inputModel = self.parameterAsEnum(parameters, "MODEL", context)
        model, backbone = [['BEiT', None], ['ConditionalDETR', None], ['Data2Vec', None], ['DETR', None], ['DPT', None], ['Mask2Former', None], ['MaskFormer', None], ['MobileNetV2', None], ['MobileViT', None], ['MobileViTV2', None], ['OneFormer', 'Swin'], ['OneFormer', 'ConvNeXT'], ['OneFormer', 'ConvNeXT2'], ['OneFormer', 'DiNAT'], ['SegFormer', None], ['UperNet', 'Swin'], ['UperNet', 'ResNet'], ['UperNet', 'ConvNeXT'], ['UperNet', 'ConvNeXT2'], ['DeepLabV3', 'MobileNet_V3_Large'], ['DeepLabV3', 'ResNet50'], ['DeepLabV3', 'ResNet101'], ['FCN', 'ResNet50'], ['FCN', 'ResNet101'], ['LRASPP', None], ['Nearest Neighbors', None], ['Logistic Regression', 'lbfgs'], ['Logistic Regression', 'liblinear'], ['Logistic Regression', 'newton-cg'], ['Logistic Regression', 'newton-cholesky'], ['Logistic Regression', 'sag'], ['Logistic Regression', 'saga'], ['Ridge', None], ['Lasso', None], ['ElasticNet', None], ['SVM', 'rbf'], ['SVM', 'linear'], ['SVM with poly kernel'], ['SVM', 'sigmoid'], ['Gaussian Process', None], ['Naive Bayes', None], ['QDA', None], ['Decision Tree', None], ['Random Forest', None], ['AdaBoost', None], ['Gradient Boosting', None], ['Multilayer Perceptron', None]][inputModel]
        inputCheckpoint = self.parameterAsFile(parameters, "CHECKPOINT", context)
        inputWeights = self.parameterAsString(parameters, "WEIGHTS", context)
        inputEpochs = self.parameterAsInt(parameters, "EPOCHS", context)
        inputBatchSize = self.parameterAsInt(parameters, "BATCH", context)
        inputLessMetrics = self.parameterAsBool(parameters, "LESS", context)
        inputLR = self.parameterAsDouble(parameters, "LR", context)
        inputMultiprocessing = self.parameterAsBool(parameters, "MULTIPROCESSING", context)
        outputDest = self.parameterAsString(parameters, self.OUTPUT, context)
        seg_train(backend, inputXTrain, inputYTrain, inputXVal, inputYVal, model, backbone, inputCheckpoint, inputWeights, inputEpochs, inputBatchSize, inputLessMetrics, inputLR, inputMultiprocessing, outputDest, feedback)
        return {"OUTPUT": outputDest}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Train semantic segmentation model'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Semantic segmentation'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def shortHelpString(self):
        return 'Trains segmentation model. \n \
        Backend: select a backend to run script. \n \
        X train tiles *.h5 files: one or more files with training tiles generated by generate_tiles() function.\n \
        Y train tiles *.h5 files: one or more files with Training tiles generated by generate_tiles() function.\n \
        X val tiles *.h5 files: one or more files with validation tiles generated by generate_tiles() function.\n \
        Y val tiles *.h5 files: one or more files with validation tiles generated by generate_tiles() function.\n \
        Model: name of model architecture. More information \
        <a href="https://remote-sensing-processor.readthedocs.io/en/latest/api/segmentation.html#list-of-available-nn-models">here</a> \
        Some of the models are not suitable for classification or regression tasks or show poor performance. \
        Choose wisely, but we recommend to start with SegFormer or UperNet.\n \
        Checkpoint of previous training: checkpoint file (*.ckpt or *.joblib) of a pre-trained model to fine-tune. \n \
        Pre-trained weights: name of pre-trained weights to fine-tune. Only works for neural networks. \
        For models from Transformers you can get available weights from \
        <a href="https://huggingface.co/models?pipeline_tag=image-segmentation&sort=downloads">Huggingface Hub</a>, \
        for Torchvision models you just set weights = True. \n \
        Number of training epochs: only works for neural networks and multilayer perceptron. \n \
        Batch size: number of training samples used in one iteration. Only works for neural networks. \n \
        Use less metrics: sometimes Torchmetrics can freeze while calculating precision, recall and IOU. \
        If it happens, try restarting with less_metrics = True. \n \
        Learning rate: learning rate of a model. Lower value results usually in better model convergence, \
        but much slower training. \n \
        Use multiprocessing: multiprocessing can significantly improve performance but also cause errors in \
        some environments. \n \
        Output model file: checkpoint file where model will be saved after training.\
        File extension must be *.ckpt for neural networks and *.joblib for scikit-learn models.\
        Model training logs are saved in the same folder.'
    
    def createInstance(self):
        return SegTrainAlgorithm()


class SegTestAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    # Constants used to refer to parameters and outputs. They will be
    # used when calling the algorithm from another algorithm, or when
    # calling from the QGIS console.

    INPUT = 'INPUT'

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "BACKEND",
                "Backend",
                options=['Python venv', 'Conda', 'Docker container'],
                defaultValue = 0
            )
        )
        
        # We add the input vector features source. It can have any kind of
        # geometry.
        self.addParameter(
            QgsProcessingParameterMultipleLayers(
                "XTEST",
                self.tr('X test tiles *.h5 files'),
                QgsProcessing.TypeFile
            )
        )
        
        self.addParameter(
            QgsProcessingParameterMultipleLayers(
                "YTEST",
                self.tr('Y test tiles *.h5 files'),
                QgsProcessing.TypeFile
            )
        )
        
        self.addParameter(
            QgsProcessingParameterFile(
                "MODEL",
                self.tr('Model file'),
                fileFilter = 'Torch or sklearn models(*.ckpt *.joblib)'
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "BATCH",
                "Batch size",
                defaultValue = 32, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "MULTIPROCESSING",
                "Use multiprocessing",
                defaultValue = True, 
                optional = True
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        backend = self.parameterAsEnum(parameters, "BACKEND", context)
        backend = ['python', 'conda', 'docker'][backend]
        inputXTest = self.parameterAsFileList(parameters, "XTEST", context)
        inputYTest = self.parameterAsFileList(parameters, "YTEST", context)
        inputModel = self.parameterAsFile(parameters, "MODEL", context)
        inputBatchSize = self.parameterAsInt(parameters, "BATCH", context)
        inputMultiprocessing = self.parameterAsBool(parameters, "MULTIPROCESSING", context)
        seg_test(backend, inputXTest, inputYTest, inputModel, inputBatchSize, inputMultiprocessing, feedback)
        return {"OUTPUT": inputModel}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Test semantic segmentation model'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Semantic segmentation'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def shortHelpString(self):
        return 'Tests segmentation model. \n \
        Backend: select a backend to run script. \n \
        X test tiles *.h5 files: one or more files with test tiles generated by generate_tiles() function.\n \
        Y test tiles *.h5 files: one or more files with test tiles generated by generate_tiles() function.\n \
        Model: Model to test. Path to model checkpoint (*.ckpt or *.joblib).\n \
        Batch size: number of training samples used in one iteration. Only works for neural networks. \n \
        Use multiprocessing: multiprocessing can significantly improve performance but also cause errors in \
        some environments. \n'
    
    def createInstance(self):
        return SegTestAlgorithm()


class SegMapAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    # Constants used to refer to parameters and outputs. They will be
    # used when calling the algorithm from another algorithm, or when
    # calling from the QGIS console.

    OUTPUT = 'OUTPUT'
    INPUT = 'INPUT'

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """
        
        self.addParameter(
            QgsProcessingParameterEnum(
                "BACKEND",
                "Backend",
                options=['Python venv', 'Conda', 'Docker container'],
                defaultValue = 0
            )
        )
        
        # We add the input vector features source. It can have any kind of
        # geometry.
        self.addParameter(
            QgsProcessingParameterMultipleLayers(
                self.INPUT,
                self.tr("Input X tiles"),
                QgsProcessing.TypeFile
            )
        )
        
        self.addParameter(
            QgsProcessingParameterRasterLayer(
                "REFERENCE",
                'Y reference raster'
            )
        )
        
        self.addParameter(
            QgsProcessingParameterFile(
                "MODEL",
                self.tr('Model file'),
                fileFilter = 'Torch or sklearn models(*.ckpt *.joblib)'
            )
        )
        
        self.addParameter(
            QgsProcessingParameterFile(
                "SAMPLES",
                self.tr('Samples file'),
                fileFilter = 'Pickle(*.pickle)'
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "NODATA",
                "Nodata value", 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                "BATCH",
                "Batch size",
                defaultValue = 32, 
                optional = True
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                "MULTIPROCESSING",
                "Use multiprocessing",
                defaultValue = True, 
                optional = True
            )
        )
        
        # We add a feature sink in which to store our processed features (this
        # usually takes the form of a newly created vector layer when the
        # algorithm is run in QGIS).
        self.addParameter(
            QgsProcessingParameterFileDestination(
                self.OUTPUT,
                self.tr('Output raster'),
                fileFilter = '.TIF(*.tif)'   
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """
        backend = self.parameterAsEnum(parameters, "BACKEND", context)
        backend = ['python', 'conda', 'docker'][backend]
        inputLayers = self.parameterAsFileList(parameters, self.INPUT, context)
        inputReference = self.parameterAsFile(parameters, "REFERENCE", context)
        inputModel = self.parameterAsFile(parameters, "MODEL", context)
        inputSamplesFile = self.parameterAsFile(parameters, "SAMPLES", context)
        inputBatchSize = self.parameterAsInt(parameters, "BATCH", context)
        inputNodata = parameters['NODATA']
        inputMultiprocessing = self.parameterAsBool(parameters, "MULTIPROCESSING", context)
        outputDest = self.parameterAsString(parameters, self.OUTPUT, context)
        seg_map(backend, inputLayers, inputReference, inputModel, inputSamplesFile, inputBatchSize, inputNodata, inputMultiprocessing, outputDest, feedback)
        return {self.OUTPUT: outputDest}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Generate map using semantic segmentation model'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Semantic segmentation'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def shortHelpString(self):
        return 'Create map using pre-trained model. \n \
        Backend: select a backend to run script. \n \
        Input X tiles: X tiled data that will be used for predictions. \
        Usually it is data generated in generate_tiles function. \
        The files order is important!!! It must be train-val-test.\n \
        Y reference raster: raster with target values which will be used as a reference raster \
        to get size, transform and crs for a map. \n \
        Model file: pre-trained model to predict target values. \
        Path to model checkpoint (*.ckpt or *.joblib).\n \
        Samples file: path to a samples .pickle file generated by generate_tiles function. \
        Nodata value: nodata value. If not defined then nodata value of y raster will be used.  \n \
        Batch size: number of training samples used in one iteration. Only works for neural networks. \n \
        Use multiprocessing: multiprocessing can significantly improve performance but also cause errors in \
        some environments. \n \
        Output raster: path where to write output map.'
    
    def createInstance(self):
        return SegMapAlgorithm()


class Ml4QgisAlgorithm(QgsProcessingAlgorithm):
    """
    This is an example algorithm that takes a vector layer and
    creates a new identical one.

    It is meant to be used as an example of how to create your own
    algorithms and explain methods and variables used to do it. An
    algorithm like this will be available in all elements, and there
    is not need for additional work.

    All Processing algorithms should extend the QgsProcessingAlgorithm
    class.
    """

    # Constants used to refer to parameters and outputs. They will be
    # used when calling the algorithm from another algorithm, or when
    # calling from the QGIS console.

    OUTPUT = 'OUTPUT'
    INPUT = 'INPUT'

    def initAlgorithm(self, config):
        """
        Here we define the inputs and output of the algorithm, along
        with some other properties.
        """

        # We add the input vector features source. It can have any kind of
        # geometry.
        self.addParameter(
            QgsProcessingParameterFeatureSource(
                self.INPUT,
                self.tr('Input layer'),
                [QgsProcessing.TypeVectorAnyGeometry]
            )
        )

        # We add a feature sink in which to store our processed features (this
        # usually takes the form of a newly created vector layer when the
        # algorithm is run in QGIS).
        self.addParameter(
            QgsProcessingParameterFeatureSink(
                self.OUTPUT,
                self.tr('Output layer')
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        """
        Here is where the processing itself takes place.
        """

        # Retrieve the feature source and sink. The 'dest_id' variable is used
        # to uniquely identify the feature sink, and must be included in the
        # dictionary returned by the processAlgorithm function.
        source = self.parameterAsSource(parameters, self.INPUT, context)
        (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT,
                context, source.fields(), source.wkbType(), source.sourceCrs())

        # Compute the number of steps to display within the progress bar and
        # get features from source
        total = 100.0 / source.featureCount() if source.featureCount() else 0
        features = source.getFeatures()

        for current, feature in enumerate(features):
            # Stop the algorithm if cancel button has been clicked
            if feedback.isCanceled():
                break

            # Add a feature in the sink
            sink.addFeature(feature, QgsFeatureSink.FastInsert)

            # Update the progress bar
            feedback.setProgress(int(current * total))

        # Return the results of the algorithm. In this case our only result is
        # the feature sink which contains the processed features, but some
        # algorithms may return multiple feature sinks, calculated numeric
        # statistics, etc. These should all be included in the returned
        # dictionary, with keys matching the feature corresponding parameter
        # or output names.
        return {self.OUTPUT: dest_id}

    def name(self):
        """
        Returns the algorithm name, used for identifying the algorithm. This
        string should be fixed for the algorithm, and must not be localised.
        The name should be unique within each provider. Names should contain
        lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Install or repair'

    def displayName(self):
        """
        Returns the translated algorithm name, which should be used for any
        user-visible display of the algorithm name.
        """
        return self.tr(self.name())

    def group(self):
        """
        Returns the name of the group this algorithm belongs to. This string
        should be localised.
        """
        return self.tr(self.groupId())

    def groupId(self):
        """
        Returns the unique ID of the group this algorithm belongs to. This
        string should be fixed for the algorithm, and must not be localised.
        The group id should be unique within each provider. Group id should
        contain lowercase alphanumeric characters only and no spaces or other
        formatting characters.
        """
        return 'Setup'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)

    def createInstance(self):
        return Ml4QgisAlgorithm()