ControlNet - Precise Image Control

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
from diffusers.utils import load_image
import torch
import numpy as np
import cv2

# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# 1. EDGE CONTROL (Canny edges)
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# First, detect edges from reference image
reference_image = load_image("your_reference.jpg")
ref_array = np.array(reference_image)

edges = cv2.Canny(ref_array, threshold1=100, threshold2=200)  # edge detection
edges_rgb = np.stack([edges, edges, edges], axis=-1)           # convert to RGB

from PIL import Image
control_image = Image.fromarray(edges_rgb)

# Load edge-conditioned ControlNet
controlnet = ControlNetModel.from_pretrained(
    "lllyasviel/control_v11p_sd15_canny",
    torch_dtype=torch.float16,
)
pipe = StableDiffusionControlNetPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    controlnet=controlnet,
    torch_dtype=torch.float16,
).to("cuda")

image = pipe(
    prompt="A beautiful oil painting of a cat, vibrant colors, masterpiece",
    image=control_image,           # control image (edges)
    controlnet_conditioning_scale=0.8,  # 0-1: how strongly to follow edges
    num_inference_steps=30,
).images[0]

# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# 2. HUMAN POSE CONTROL (OpenPose)
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# pip install controlnet-aux
from controlnet_aux import OpenposeDetector

pose_detector = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
pose_image = pose_detector(load_image("person_photo.jpg"))  # detect body keypoints

controlnet_pose = ControlNetModel.from_pretrained(
    "lllyasviel/control_v11p_sd15_openpose",
    torch_dtype=torch.float16,
)
pipe_pose = StableDiffusionControlNetPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5", controlnet=controlnet_pose, torch_dtype=torch.float16
).to("cuda")

# Generate character in same pose but different style/clothing
generated = pipe_pose(
    prompt="A samurai warrior in traditional armor, cinematic lighting",
    negative_prompt="blurry, ugly, bad anatomy",
    image=pose_image,                     # OpenPose skeleton
    controlnet_conditioning_scale=1.0,    # strong pose adherence
    guidance_scale=8,
    num_inference_steps=30,
).images[0]

# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# CONTROLNET TYPES -- what each controls
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
controlnet_types = {
    "Canny":        "Explicit edge map -- control exact contours and shapes",
    "OpenPose":     "Human body skeleton -- control exact body pose",
    "Depth":        "Depth map -- control 3D spatial layout",
    "Segmentation": "Semantic map -- control foreground/background layout",
    "Normal Map":   "Surface normals -- control detailed 3D surface detail",
    "Scribble":     "Rough sketch -- turn rough drawings into realistic images",
    "IP-Adapter":   "Image prompt: generate images with same style/content as reference",
}

for control_type, description in controlnet_types.items():
    print(f"  {control_type:15s}: {description}")