Support inpaint and outpaint when call image prompt

examples/examples.py

@@ -45,7 +45,7 @@ class Config():
 
 imgs_base_path = os.path.join(os.path.dirname(__file__), 'imgs')
 
-with open(os.path.join(imgs_base_path, "1485005453352708.jpeg"), "rb") as f:
+with open(os.path.join(imgs_base_path, "bear.jpg"), "rb") as f:
     img1 = f.read()
     image_base64 = base64.b64encode(img1).decode('utf-8')  
     f.close()
@@ -92,7 +92,6 @@ def image_prompt(img_prompt: list, params: dict) -> dict:
     """
     Image Prompt
     """
-    params["prompt"] = "cat"
     params["image_prompts"] = img_prompt
     data = json.dumps(params)
     response = requests.post(url=f"{cfg.fooocus_host}{cfg.img_prompt}",
@@ -101,20 +100,31 @@ def image_prompt(img_prompt: list, params: dict) -> dict:
                         timeout=300)
     return response.json()
 
+def image_prompt_with_inpaint(img_prompt: list, input_image: str, input_mask: str, params: dict) -> dict:
+    """
+    Image Prompt
+    """
+    params["image_prompts"] = img_prompt
+    params["input_image"] = input_image
+    params["input_mask"] = input_mask
+    params["outpaint_selections"] = ["Left", "Right"]
+    data = json.dumps(params)
+    response = requests.post(url=f"{cfg.fooocus_host}{cfg.img_prompt}",
+                        data=data,
+                        headers=headers,
+                        timeout=300)
+    return response.json()
+
 
 img_prompt = [
     {
         "cn_img": image_base64,
         "cn_stop": 0.6,
         "cn_weight": 0.6,
         "cn_type": "ImagePrompt"
-    },{
-        "cn_img": s_base64,
-        "cn_stop": 0.6,
-        "cn_weight": 0.6,
-        "cn_type": "ImagePrompt"
     }
 ]
 # print(upscale_vary(image=image_base64))
 # print(inpaint_outpaint(input_image=s_base64, input_mask=m_base64))
-print(image_prompt(img_prompt=img_prompt, params=img_prompt_params))
+# print(image_prompt(img_prompt=img_prompt, params=img_prompt_params))
+print(image_prompt_with_inpaint(img_prompt=img_prompt, input_image=s_base64, input_mask=m_base64, params=img_prompt_params))

examples/examples_v1.py

@@ -112,7 +112,7 @@ def image_prompt_v2(params: ImagePromptParamsJson = ImagePromptParamsJson()) ->
 if __name__ == '__main__':
     imgs_base_path = os.path.join(os.path.dirname(__file__), 'imgs')
 
-    input_image = open(os.path.join(imgs_base_path,'1485005453352708.jpeg'), 'rb').read()
+    input_image = open(os.path.join(imgs_base_path,'bear.jpg'), 'rb').read()
     input_source = open(os.path.join(imgs_base_path,'s.jpg'), 'rb').read()
     input_mask = open(os.path.join(imgs_base_path,'m.png'), 'rb').read()
 

fooocusapi/api.py

@@ -61,10 +61,10 @@ def call_worker(req: Text2ImgRequest, accept: str):
     task_type = TaskType.text_2_img
     if isinstance(req, ImgUpscaleOrVaryRequest) or isinstance(req, ImgUpscaleOrVaryRequestJson):
         task_type = TaskType.img_uov
-    elif isinstance(req, ImgInpaintOrOutpaintRequest) or isinstance(req, ImgInpaintOrOutpaintRequestJson):
-        task_type = TaskType.img_inpaint_outpaint
     elif isinstance(req, ImgPromptRequest) or isinstance(req, ImgPromptRequestJson):
         task_type = TaskType.img_prompt
+    elif isinstance(req, ImgInpaintOrOutpaintRequest) or isinstance(req, ImgInpaintOrOutpaintRequestJson):
+        task_type = TaskType.img_inpaint_outpaint
 
     params = req_to_params(req)
     queue_task = task_queue.add_task(
@@ -215,6 +215,11 @@ def img_prompt(req: ImgPromptRequestJson,
         req.image_number = 1
     else:
         streaming_output = False
+
+    if req.input_image is not None:
+        req.input_image = base64_to_stream(req.input_image)
+    if req.input_mask is not None:
+        req.input_mask = base64_to_stream(req.input_mask)
 
     default_image_promt = ImagePrompt(cn_img=None)
     image_prompts_files: List[ImagePrompt] = []

fooocusapi/api_utils.py

@@ -79,6 +79,10 @@ def req_to_params(req: Text2ImgRequest) -> ImageGenerationParams:
 
     image_prompts = []
     if isinstance(req, ImgPromptRequest) or isinstance(req, ImgPromptRequestJson):
+        # Auto set mixing_image_prompt_and_inpaint to True
+        if len(req.image_prompts) > 0 and req.input_image is not None and req.advanced_params is not None:
+            req.advanced_params.mixing_image_prompt_and_inpaint = True
+
         for img_prompt in req.image_prompts:
             if img_prompt.cn_img is not None:
                 cn_img = read_input_image(img_prompt.cn_img)

fooocusapi/models.py

@@ -195,10 +195,10 @@ def as_form(cls, input_image: UploadFile = Form(description="Init image for inpa
                 outpaint_selections: List[str] = Form(
                     [], description="Outpaint expansion selections, literal 'Left', 'Right', 'Top', 'Bottom' seperated by comma"),
 
-                outpaint_distance_left: int = Form(default=0, description="Set outpaint left distance, 0 for default"),
-                outpaint_distance_right: int = Form(default=0, description="Set outpaint right distance, 0 for default"),
-                outpaint_distance_top: int = Form(default=0, description="Set outpaint top distance, 0 for default"),
-                outpaint_distance_bottom: int = Form(default=0, description="Set outpaint bottom distance, 0 for default"),
+                outpaint_distance_left: int = Form(default=0, description="Set outpaint left distance, -1 for default"),
+                outpaint_distance_right: int = Form(default=0, description="Set outpaint right distance, -1 for default"),
+                outpaint_distance_top: int = Form(default=0, description="Set outpaint top distance, -1 for default"),
+                outpaint_distance_bottom: int = Form(default=0, description="Set outpaint bottom distance, -1 for default"),
                 prompt: str = Form(''),
                 negative_prompt: str = Form(default_prompt_negative),
                 style_selections: List[str] = Form(defualt_styles, description="Fooocus style selections, seperated by comma"),
@@ -257,7 +257,7 @@ def as_form(cls, input_image: UploadFile = Form(description="Init image for inpa
                 errs = ve.errors()
                 raise RequestValidationError(errors=[errs])
 
-        return cls(input_image=input_image, input_mask=input_mask, inpaint_additional_prompt=inpaint_additional_prompt,outpaint_selections=outpaint_selections_arr,
+        return cls(input_image=input_image, input_mask=input_mask, inpaint_additional_prompt=inpaint_additional_prompt, outpaint_selections=outpaint_selections_arr,
                    outpaint_distance_left=outpaint_distance_left, outpaint_distance_right=outpaint_distance_right, outpaint_distance_top=outpaint_distance_top, outpaint_distance_bottom=outpaint_distance_bottom,
                    prompt=prompt, negative_prompt=negative_prompt, style_selections=style_selection_arr,
                    performance_selection=performance_selection, aspect_ratios_selection=aspect_ratios_selection,
@@ -266,11 +266,19 @@ def as_form(cls, input_image: UploadFile = Form(description="Init image for inpa
                    loras=loras_model, advanced_params=advanced_params_obj, require_base64=require_base64, async_process=async_process)
 
 
-class ImgPromptRequest(Text2ImgRequest):
+class ImgPromptRequest(ImgInpaintOrOutpaintRequest):
     image_prompts: List[ImagePrompt]
 
     @classmethod
-    def as_form(cls, cn_img1: UploadFile = Form(File(None), description="Input image for image prompt"),
+    def as_form(cls, input_image: UploadFile = Form(Field(None), description="Init image for inpaint or outpaint"),
+                input_mask: UploadFile = Form(File(None), description="Inpaint or outpaint mask"),
+                inpaint_additional_prompt: str | None = Form(None, description="Describe what you want to inpaint"),
+                outpaint_selections: List[str] = Form([], description="Outpaint expansion selections, literal 'Left', 'Right', 'Top', 'Bottom' seperated by comma"),
+                outpaint_distance_left: int = Form(default=0, description="Set outpaint left distance, 0 for default"),
+                outpaint_distance_right: int = Form(default=0, description="Set outpaint right distance, 0 for default"),
+                outpaint_distance_top: int = Form(default=0, description="Set outpaint top distance, 0 for default"),
+                outpaint_distance_bottom: int = Form(default=0, description="Set outpaint bottom distance, 0 for default"),
+                cn_img1: UploadFile = Form(File(None), description="Input image for image prompt"),
                 cn_stop1: float | None = Form(
                     default=None, ge=0, le=1, description="Stop at for image prompt, None for default value"),
                 cn_weight1: float | None = Form(
@@ -320,6 +328,10 @@ def as_form(cls, cn_img1: UploadFile = Form(File(None), description="Input image
                 require_base64: bool = Form(default=False, description="Return base64 data of generated image"),
                 async_process: bool = Form(default=False, description="Set to true will run async and return job info for retrieve generataion result later"),
                 ):
+        if isinstance(input_image, File):
+            input_image = None
+        if isinstance(input_mask, File):
+            input_mask = None
         if isinstance(cn_img1, File):
             cn_img1 = None
         if isinstance(cn_img2, File):
@@ -329,6 +341,18 @@ def as_form(cls, cn_img1: UploadFile = Form(File(None), description="Input image
         if isinstance(cn_img4, File):
             cn_img4 = None
 
+        outpaint_selections_arr: List[OutpaintExpansion] = []
+        for part in outpaint_selections:
+            if len(part) > 0:
+                for s in part.split(','):
+                    try:
+                        expansion = OutpaintExpansion(s)
+                        outpaint_selections_arr.append(expansion)
+                    except ValueError as ve:
+                        err = InitErrorDetails(type='enum', loc=['outpaint_selections'], input=outpaint_selections, ctx={
+                            'expected': "Literal 'Left', 'Right', 'Top', 'Bottom' seperated by comma"})
+                        raise RequestValidationError(errors=[err])
+
         image_prompts: List[ImagePrompt] = []
         image_prompt_config = [(cn_img1, cn_stop1, cn_weight1, cn_type1), (cn_img2, cn_stop2, cn_weight2, cn_type2),
                                (cn_img3, cn_stop3, cn_weight3, cn_type3), (cn_img4, cn_stop4, cn_weight4, cn_type4)]
@@ -360,7 +384,9 @@ def as_form(cls, cn_img1: UploadFile = Form(File(None), description="Input image
                 errs = ve.errors()
                 raise RequestValidationError(errors=[errs])
 
-        return cls(image_prompts=image_prompts, prompt=prompt, negative_prompt=negative_prompt, style_selections=style_selection_arr,
+        return cls(input_image=input_image, input_mask=input_mask, inpaint_additional_prompt=inpaint_additional_prompt, outpaint_selections=outpaint_selections_arr,
+                   outpaint_distance_left=outpaint_distance_left, outpaint_distance_right=outpaint_distance_right, outpaint_distance_top=outpaint_distance_top, outpaint_distance_bottom=outpaint_distance_bottom,
+                   image_prompts=image_prompts, prompt=prompt, negative_prompt=negative_prompt, style_selections=style_selection_arr,
                    performance_selection=performance_selection, aspect_ratios_selection=aspect_ratios_selection,
                    image_number=image_number, image_seed=image_seed, sharpness=sharpness, guidance_scale=guidance_scale,
                    base_model_name=base_model_name, refiner_model_name=refiner_model_name, refiner_switch=refiner_switch,

fooocusapi/models_v2.py

@@ -25,5 +25,6 @@ class ImagePromptJson(BaseModel):
     cn_type: ControlNetType = Field(default=ControlNetType.cn_ip, description="ControlNet type for image prompt")
 
 
-class ImgPromptRequestJson(Text2ImgRequest):
+class ImgPromptRequestJson(ImgInpaintOrOutpaintRequestJson):
+    input_image: str | None = Field(None, description="Init image for inpaint or outpaint as base64")
     image_prompts: List[ImagePromptJson | ImagePrompt]

fooocusapi/parameters.py

@@ -159,6 +159,10 @@ def __init__(self, prompt: str,
             inpaint_strength = 1.0
             inpaint_respective_field = 0.618
 
+            # Auto set mixing_image_prompt_and_inpaint to True
+            if len(self.image_prompts) > 0 and inpaint_input_image is not None:
+                mixing_image_prompt_and_inpaint = True
+
             self.advanced_params = [
                 disable_preview, adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg, sampler_name, \
                 scheduler_name, generate_image_grid, overwrite_step, overwrite_switch, overwrite_width, overwrite_height, \

fooocusapi/worker.py

@@ -121,7 +121,7 @@ def yield_result(_, imgs, tasks):
         refiner_switch = params.refiner_switch
         loras = params.loras
         input_image_checkbox = params.uov_input_image is not None or params.inpaint_input_image is not None or len(params.image_prompts) > 0
-        current_tab = 'uov' if params.uov_method != flags.disabled else 'inpaint' if params.inpaint_input_image is not None else 'ip' if len(params.image_prompts) > 0 else None
+        current_tab = 'uov' if params.uov_method != flags.disabled else 'ip' if len(params.image_prompts) > 0 else 'inpaint' if params.inpaint_input_image is not None else None
         uov_method = params.uov_method
         upscale_value = params.upscale_value
         uov_input_image = params.uov_input_image