This project was set to be a for lower end cards with sufficient VRAM (2070S 8GB in this case). It is possible to generate images with even smaller amounts of VRAM (6GB), but I wanted to confirm a simple walk through for anyone interested in running image generation from home.
All images, gifs, and videos that were used here were used with open source models with usage for research applications. All images, gifs, and videos used here, unless specifically stated otherwise, were made locally using ComfyUI as our UX.
It is still a work in progress and will be updated when SD3 comes out.
ToDo: Add workflows for SVD img2video and 3d img2model.
Tips while getting started:
You can drag and drop the workflow image (last image in the list below) on to your own ComfyUI project to copy it. Or download one of the.jsons in the project files and load it with your comfy UI manager. Doing this can let you play with the same settings I'm using. I'll have a Tutorial for getting into the basics of ComfyUI soon.
This project outlines more than image production. We will also show that not only can images be produced, but stable video through AnimateDiff, and 3D modeling through TripoSR. This project serves to demonstrate that even on a modest card, Stable Diffusion has a lot of potential to give.
Examples and Images
1 / 5 • Example 1.1 Shaun a real picture taken on a walk
1 / 2 • Example 2
example 2.1 AnimateDiff Animation with panning lora
example 3.1. using a scribe to generate a 3d model .obj
example 3.2 using a puppy to generate a very poorly shaped puppy
This is a workflow.json file. In ComfyUI, you can load these with the ComfyUI manager to import someone else' s work flow. You can then use the manager to install missing custom nodes, and then missing models with its model manager, all from the UI interface.
{"last_node_id":78,"last_link_id":223,"nodes":[{"id":63,"type":"VAEEncode","pos":[547,636],"size":{"0":210,"1":46},"flags":{},"order":7,"mode":0,"inputs":[{"name":"pixels","type":"IMAGE","link":177},{"name":"vae","type":"VAE","link":178}],"outputs":[{"name":"LATENT","type":"LATENT","links":[180],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"VAEEncode"}},{"id":68,"type":"Canny","pos":[92,491],"size":{"0":315,"1":82},"flags":{},"order":4,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":192}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[195],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Canny"},"widgets_values":[0.09,0.09]},{"id":8,"type":"VAEDecode","pos":[1004,81],"size":{"0":210,"1":46},"flags":{},"order":11,"mode":0,"inputs":[{"name":"samples","type":"LATENT","link":86},{"name":"vae","type":"VAE","link":66}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[25],"slot_index":0}],"properties":{"Node name for S&R":"VAEDecode"}},{"id":73,"type":"Upscale Model Loader","pos":[389,1008],"size":{"0":315,"1":78},"flags":{},"order":0,"mode":0,"outputs":[{"name":"UPSCALE_MODEL","type":"UPSCALE_MODEL","links":[],"shape":3},{"name":"MODEL_NAME_TEXT","type":"STRING","links":null,"shape":3}],"properties":{"Node name for S&R":"Upscale Model Loader"},"widgets_values":["4x-UltraSharp.pth"]},{"id":76,"type":"VAEDecode","pos":[927,1325],"size":{"0":210,"1":46},"flags":{},"order":14,"mode":0,"inputs":[{"name":"samples","type":"LATENT","link":223},{"name":"vae","type":"VAE","link":212,"slot_index":1}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[213],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"VAEDecode"}},{"id":74,"type":"SaveImage","pos":[1398,1325],"size":{"0":2112.820556640625,"1":1124.08642578125},"flags":{},"order":15,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":213}],"properties":{},"widgets_values":["ComfyUI"]},{"id":62,"type":"LoadImage","pos":[-665,561],"size":{"0":672.0782470703125,"1":764.6511840820312},"flags":{},"order":1,"mode":0,"outputs":[{"name":"IMAGE","type":"IMAGE","links":[177,192],"shape":3,"slot_index":0},{"name":"MASK","type":"MASK","links":null,"shape":3,"slot_index":1}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["shaunathan_danials_cropped_2.jpg","image"]},{"id":54,"type":"Power Prompt (rgthree)","pos":[560,342],"size":{"0":210,"1":194},"flags":{},"order":6,"mode":0,"inputs":[{"name":"opt_model","type":"MODEL","link":189,"dir":3},{"name":"opt_clip","type":"CLIP","link":188,"dir":3}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[161,221],"shape":3,"dir":4,"disabled":false,"slot_index":0},{"name":"MODEL","type":"MODEL","links":null,"shape":3,"dir":4,"disabled":false,"slot_index":1},{"name":"CLIP","type":"CLIP","links":null,"shape":3,"dir":4,"disabled":false},{"name":"TEXT","type":"STRING","links":null,"shape":3,"dir":4,"color_off":"#7F7","color_on":"#7F7"}],"properties":{"Node name for S&R":"Power Prompt (rgthree)","combos_filter":""},"widgets_values":["","CHOOSE","CHOOSE"]},{"id":9,"type":"SaveImage","pos":[1461,34],"size":{"0":1851.832763671875,"1":1204.7490234375},"flags":{},"order":13,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":25}],"properties":{},"widgets_values":["Test_LoRA"]},{"id":69,"type":"ControlNetLoader","pos":[45,686],"size":{"0":315,"1":58},"flags":{},"order":2,"mode":0,"outputs":[{"name":"CONTROL_NET","type":"CONTROL_NET","links":[193],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"ControlNetLoader"},"widgets_values":["diffusers_xl_canny_full.safetensors"]},{"id":56,"type":"Power Prompt (rgthree)","pos":[497,-23],"size":{"0":309.2365417480469,"1":297.9857482910156},"flags":{},"order":5,"mode":0,"inputs":[{"name":"opt_model","type":"MODEL","link":186,"dir":3},{"name":"opt_clip","type":"CLIP","link":187,"dir":3}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[196,220],"shape":3,"dir":4,"disabled":false,"slot_index":0},{"name":"MODEL","type":"MODEL","links":[168,222],"shape":3,"dir":4,"disabled":false,"slot_index":1},{"name":"CLIP","type":"CLIP","links":null,"shape":3,"dir":4,"disabled":false},{"name":"TEXT","type":"STRING","links":null,"shape":3,"dir":4,"color_off":"#7F7","color_on":"#7F7"}],"properties":{"Node name for S&R":"Power Prompt (rgthree)","combos_filter":""},"widgets_values":["<lora:SDXL\\XL_Bosstyle:1.0>, bear monster chained down in combat, Bossstyle ","CHOOSE","CHOOSE"]},{"id":64,"type":"RepeatLatentBatch","pos":[862,632],"size":{"0":315,"1":58},"flags":{},"order":9,"mode":0,"inputs":[{"name":"samples","type":"LATENT","link":180}],"outputs":[{"name":"LATENT","type":"LATENT","links":[181],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"RepeatLatentBatch"},"widgets_values":[3]},{"id":4,"type":"CheckpointLoaderSimple","pos":[20,76],"size":{"0":315,"1":98},"flags":{},"order":3,"mode":0,"outputs":[{"name":"MODEL","type":"MODEL","links":[186,189],"slot_index":0},{"name":"CLIP","type":"CLIP","links":[187,188],"slot_index":1},{"name":"VAE","type":"VAE","links":[66,178,212],"slot_index":2}],"properties":{"Node name for S&R":"CheckpointLoaderSimple"},"widgets_values":["SDXL\\Lightning\\lightningFusionSDXL_v12.safetensors"]},{"id":78,"type":"KSampler","pos":[907,854],"size":{"0":315,"1":262},"flags":{},"order":12,"mode":0,"inputs":[{"name":"model","type":"MODEL","link":222},{"name":"positive","type":"CONDITIONING","link":220},{"name":"negative","type":"CONDITIONING","link":221},{"name":"latent_image","type":"LATENT","link":218}],"outputs":[{"name":"LATENT","type":"LATENT","links":[223],"slot_index":0}],"properties":{"Node name for S&R":"KSampler"},"widgets_values":[669435105453608,"randomize",4,1,"dpmpp_sde","karras",0.25]},{"id":3,"type":"KSampler","pos":[958,262],"size":{"0":315,"1":262},"flags":{},"order":10,"mode":0,"inputs":[{"name":"model","type":"MODEL","link":168},{"name":"positive","type":"CONDITIONING","link":197},{"name":"negative","type":"CONDITIONING","link":161},{"name":"latent_image","type":"LATENT","link":181}],"outputs":[{"name":"LATENT","type":"LATENT","links":[86,218],"slot_index":0}],"properties":{"Node name for S&R":"KSampler"},"widgets_values":[914576475910997,"randomize",4,2,"dpmpp_sde","karras",1]},{"id":70,"type":"ControlNetApply","pos":[395,738],"size":{"0":295.659423828125,"1":160.42420959472656},"flags":{},"order":8,"mode":0,"inputs":[{"name":"conditioning","type":"CONDITIONING","link":196},{"name":"control_net","type":"CONTROL_NET","link":193},{"name":"image","type":"IMAGE","link":195}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[197],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"ControlNetApply"},"widgets_values":[0.3]}],"links":[[25,8,0,9,0,"IMAGE"],[66,4,2,8,1,"VAE"],[86,3,0,8,0,"LATENT"],[161,54,0,3,2,"CONDITIONING"],[168,56,1,3,0,"MODEL"],[177,62,0,63,0,"IMAGE"],[178,4,2,63,1,"VAE"],[180,63,0,64,0,"LATENT"],[181,64,0,3,3,"LATENT"],[186,4,0,56,0,"MODEL"],[187,4,1,56,1,"CLIP"],[188,4,1,54,1,"CLIP"],[189,4,0,54,0,"MODEL"],[192,62,0,68,0,"IMAGE"],[193,69,0,70,1,"CONTROL_NET"],[195,68,0,70,2,"IMAGE"],[196,56,0,70,0,"CONDITIONING"],[197,70,0,3,1,"CONDITIONING"],[212,4,2,76,1,"VAE"],[213,76,0,74,0,"IMAGE"],[218,3,0,78,3,"LATENT"],[220,56,0,78,1,"CONDITIONING"],[221,54,0,78,2,"CONDITIONING"],[222,56,1,78,0,"MODEL"],[223,78,0,76,0,"LATENT"]],"groups":[],"config":{},"extra":{},"version":0.4}
Comments
Please log in or sign up to comment.