Merge dac0710c88 into 22ad513c72

* add dependency aware cache that removed a cached node as soon as all of its decendents have executed. This allows users with lower RAM to run workflows they would otherwise not be able to run. The downside is that every workflow will fully run each time even if no nodes have changed.

* remove test code

* tidy code

.dockerignore

@@ -0,0 +1,8 @@
+.*
+!.git
+__pycache__/
+*.py[cod]
+input
+models
+notebooks
+output

.github/workflows/docker.yml

@@ -0,0 +1,101 @@
+name: Build and publish Docker images
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  release:
+    types: [published]
+
+jobs:
+  docker:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - id: cu118
+            name: CUDA 11.8
+            pytorch_install_args: "--index-url https://download.pytorch.org/whl/cu118"
+          - id: cu121
+            name: CUDA 12.1
+            pytorch_install_args: "--index-url https://download.pytorch.org/whl/cu121"
+          - id: cu124
+            name: CUDA 12.4
+            pytorch_install_args: "--index-url https://download.pytorch.org/whl/cu124"
+          - id: rocm6.2
+            name: ROCm 6.2
+            pytorch_install_args: "--index-url https://download.pytorch.org/whl/rocm6.2"
+          - id: cpu
+            name: CPU only
+            pytorch_install_args: "--index-url https://download.pytorch.org/whl/cpu"
+            extra_args: --cpu
+
+
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Check which repositories to use
+        id: repositories
+        run: |
+          echo "GHCR_IMAGE_NAME=ghcr.io/${GITHUB_REPOSITORY_OWNER}/comfyui" >> "$GITHUB_ENV"
+          if [[ -n "${DOCKERHUB_USERNAME}" ]]; then
+            echo "DOCKERHUB_IMAGE_NAME=${DOCKERHUB_USERNAME}/comfyui" >> "$GITHUB_ENV"
+          else
+            echo "DOCKERHUB_IMAGE_NAME=" >> "$GITHUB_ENV"
+            echo "No Docker Hub username set, only deploying to GitHub Container Repository"
+          fi
+        env:
+            DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+            GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            ${{ env.DOCKERHUB_IMAGE_NAME }}
+            ${{ env.GHCR_IMAGE_NAME }}
+          flavor: |
+            suffix=-${{ matrix.id }},onlatest=true
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+            # set latest tag for default branch
+            type=raw,value=latest,enable=${{ github.event_name == 'release' }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request' && env.DOCKERHUB_IMAGE_NAME != ''
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            PYTORCH_INSTALL_ARGS=${{ matrix.pytorch_install_args }}
+            EXTRA_ARGS=${{ matrix.extra_args }}
+          cache-from: type=gha,scope=${{ github.ref_name }}-${{ matrix.id }}
+          cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-${{ matrix.id }}

Dockerfile

@@ -0,0 +1,79 @@
+# syntax=docker/dockerfile:1
+
+ARG PYTHON_VERSION=3.12
+
+FROM python:${PYTHON_VERSION}-slim
+
+ARG PYTORCH_INSTALL_ARGS=""
+ARG EXTRA_ARGS=""
+ARG USERNAME=comfyui
+ARG USER_UID=1000
+ARG USER_GID=${USER_UID}
+
+# Fail fast on errors or unset variables
+SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
+
+RUN <<EOF
+	groupadd --gid ${USER_GID} ${USERNAME}
+	useradd --uid ${USER_UID} --gid ${USER_GID} -m ${USERNAME}
+EOF
+
+RUN <<EOF
+	apt-get update
+	apt-get install -y --no-install-recommends \
+		git \
+		git-lfs \
+		rsync \
+		fonts-recommended
+EOF
+
+# run instructions as user
+USER ${USER_UID}:${USER_GID}
+
+WORKDIR /app
+
+ENV XDG_CACHE_HOME=/cache
+ENV PIP_CACHE_DIR=/cache/pip
+ENV VIRTUAL_ENV=/app/venv
+ENV VIRTUAL_ENV_CUSTOM=/app/custom_venv
+
+# create cache directory. During build we will use a cache mount,
+# but later this is useful for custom node installs
+RUN --mount=type=cache,target=/cache/,uid=${USER_UID},gid=${USER_GID} \
+	mkdir -p ${PIP_CACHE_DIR}
+
+# create virtual environment to manage packages
+RUN python -m venv ${VIRTUAL_ENV}
+
+# run python from venv (prefer custom_venv over baked-in one)
+ENV PATH="${VIRTUAL_ENV_CUSTOM}/bin:${VIRTUAL_ENV}/bin:${PATH}"
+
+RUN --mount=type=cache,target=/cache/,uid=${USER_UID},gid=${USER_GID} \
+	pip install torch torchvision torchaudio ${PYTORCH_INSTALL_ARGS}
+
+# copy requirements files first so packages can be cached separately
+COPY --chown=${USER_UID}:${USER_GID} requirements.txt .
+RUN --mount=type=cache,target=/cache/,uid=${USER_UID},gid=${USER_GID} \
+	pip install -r requirements.txt
+
+# Not strictly required for comfyui, but prevents non-working variants of
+# cv2 being pulled in by custom nodes
+RUN --mount=type=cache,target=/cache/,uid=${USER_UID},gid=${USER_GID} \
+	pip install opencv-python-headless
+
+COPY --chown=${USER_UID}:${USER_GID} . .
+
+COPY --chown=nobody:${USER_GID} .git .git
+
+# default environment variables
+ENV COMFYUI_ADDRESS=0.0.0.0
+ENV COMFYUI_PORT=8188
+ENV COMFYUI_EXTRA_BUILD_ARGS="${EXTRA_ARGS}"
+ENV COMFYUI_EXTRA_ARGS=""
+# default start command
+CMD \
+	if [ -d "${VIRTUAL_ENV_CUSTOM}" ]; then \
+		rsync -aP "${VIRTUAL_ENV}/" "${VIRTUAL_ENV_CUSTOM}/" ;\
+		sed -i "s!${VIRTUAL_ENV}!${VIRTUAL_ENV_CUSTOM}!g" "${VIRTUAL_ENV_CUSTOM}/pyvenv.cfg" ;\
+	fi ;\
+	python -u main.py --listen "${COMFYUI_ADDRESS}" --port "${COMFYUI_PORT}" ${COMFYUI_EXTRA_BUILD_ARGS} ${COMFYUI_EXTRA_ARGS}

README.md

@@ -237,6 +237,45 @@ Install the dependencies by opening your terminal inside the ComfyUI folder and:
 
 After this you should have everything installed and can proceed to running ComfyUI.
 
+
+## Docker
+
+There are prebuilt docker images for AMD and NVIDIA GPUs on [GitHub Packages](https://ghcr.io/comfyanonymous/comfyui).
+
+You can pull them to your local docker registry with:
+
+```shell
+# For NVIDIA GPUs
+docker pull ghcr.io/comfyanonymous/comfyui:latest-cu124
+# For AMD GPUs
+docker pull ghcr.io/comfyanonymous/comfyui:latest-rocm6.2
+
+# For CPU only
+docker pull ghcr.io/comfyanonymous/comfyui:latest-cpu
+```
+
+### Building images manually
+
+You can build a docker image with the `Dockerfile` in this repo.
+Specify, `PYTORCH_INSTALL_ARGS` build arg with one of the PyTorch commands above to build for AMD or NVIDIA GPUs.
+```shell
+docker build --build-arg PYTORCH_INSTALL_ARGS="--index-url https://download.pytorch.org/whl/cu122" .
+```
+```shell
+docker build --build-arg PYTORCH_INSTALL_ARGS="--index-url https://download.pytorch.org/whl/rocm6.2" .
+```
+The `Dockerfile` requires BuildKit to be enabled. If your Docker does not support the buildx command, you can
+enable BuildKit by setting the `DOCKER_BUILDKIT` environment variable.
+```shell
+DOCKER_BUILDKIT=1 docker build --build-arg PYTORCH_INSTALL_ARGS="--index-url https://download.pytorch.org/whl/cu124" .
+```
+> [!NOTE]
+> For building the CPU-only image, it is recommended that you add the --cpu flag to the EXTRA_ARGS build arg:
+>
+> ```shell
+> docker build --build-arg PYTORCH_INSTALL_ARGS="--index-url https://download.pytorch.org/whl/cpu" --build-arg EXTRA_ARGS=--cpu .
+> ```
+
 ### Others:
 
 #### Apple Mac silicon

comfy/cli_args.py

@@ -101,6 +101,7 @@ parser.add_argument("--preview-size", type=int, default=512, help="Sets the maxi
 cache_group = parser.add_mutually_exclusive_group()
 cache_group.add_argument("--cache-classic", action="store_true", help="Use the old style (aggressive) caching.")
 cache_group.add_argument("--cache-lru", type=int, default=0, help="Use LRU caching with a maximum of N node results cached. May use more RAM/VRAM.")
+cache_group.add_argument("--cache-none", action="store_true", help="Reduced RAM/VRAM usage at the expense of executing every node for each run.")
 
 attn_group = parser.add_mutually_exclusive_group()
 attn_group.add_argument("--use-split-cross-attention", action="store_true", help="Use the split cross attention optimization. Ignored when xformers is used.")

comfy_execution/caching.py

@@ -316,3 +316,156 @@ class LRUCache(BasicCache):
             self.children[cache_key].append(self.cache_key_set.get_data_key(child_id))
         return self
 
+
+class DependencyAwareCache(BasicCache):
+    """
+    A cache implementation that tracks dependencies between nodes and manages
+    their execution and caching accordingly. It extends the BasicCache class.
+    Nodes are removed from this cache once all of their descendants have been
+    executed.
+    """
+
+    def __init__(self, key_class):
+        """
+        Initialize the DependencyAwareCache.
+
+        Args:
+            key_class: The class used for generating cache keys.
+        """
+        super().__init__(key_class)
+        self.descendants = {}  # Maps node_id -> set of descendant node_ids
+        self.ancestors = {}    # Maps node_id -> set of ancestor node_ids
+        self.executed_nodes = set()  # Tracks nodes that have been executed
+
+    def set_prompt(self, dynprompt, node_ids, is_changed_cache):
+        """
+        Clear the entire cache and rebuild the dependency graph.
+
+        Args:
+            dynprompt: The dynamic prompt object containing node information.
+            node_ids: List of node IDs to initialize the cache for.
+            is_changed_cache: Flag indicating if the cache has changed.
+        """
+        # Clear all existing cache data
+        self.cache.clear()
+        self.subcaches.clear()
+        self.descendants.clear()
+        self.ancestors.clear()
+        self.executed_nodes.clear()
+
+        # Call the parent method to initialize the cache with the new prompt
+        super().set_prompt(dynprompt, node_ids, is_changed_cache)
+
+        # Rebuild the dependency graph
+        self._build_dependency_graph(dynprompt, node_ids)
+
+    def _build_dependency_graph(self, dynprompt, node_ids):
+        """
+        Build the dependency graph for all nodes.
+
+        Args:
+            dynprompt: The dynamic prompt object containing node information.
+            node_ids: List of node IDs to build the graph for.
+        """
+        self.descendants.clear()
+        self.ancestors.clear()
+        for node_id in node_ids:
+            self.descendants[node_id] = set()
+            self.ancestors[node_id] = set()
+
+        for node_id in node_ids:
+            inputs = dynprompt.get_node(node_id)["inputs"]
+            for input_data in inputs.values():
+                if is_link(input_data):  # Check if the input is a link to another node
+                    ancestor_id = input_data[0]
+                    self.descendants[ancestor_id].add(node_id)
+                    self.ancestors[node_id].add(ancestor_id)
+
+    def set(self, node_id, value):
+        """
+        Mark a node as executed and store its value in the cache.
+
+        Args:
+            node_id: The ID of the node to store.
+            value: The value to store for the node.
+        """
+        self._set_immediate(node_id, value)
+        self.executed_nodes.add(node_id)
+        self._cleanup_ancestors(node_id)
+
+    def get(self, node_id):
+        """
+        Retrieve the cached value for a node.
+
+        Args:
+            node_id: The ID of the node to retrieve.
+
+        Returns:
+            The cached value for the node.
+        """
+        return self._get_immediate(node_id)
+
+    def ensure_subcache_for(self, node_id, children_ids):
+        """
+        Ensure a subcache exists for a node and update dependencies.
+
+        Args:
+            node_id: The ID of the parent node.
+            children_ids: List of child node IDs to associate with the parent node.
+
+        Returns:
+            The subcache object for the node.
+        """
+        subcache = super()._ensure_subcache(node_id, children_ids)
+        for child_id in children_ids:
+            self.descendants[node_id].add(child_id)
+            self.ancestors[child_id].add(node_id)
+        return subcache
+
+    def _cleanup_ancestors(self, node_id):
+        """
+        Check if ancestors of a node can be removed from the cache.
+
+        Args:
+            node_id: The ID of the node whose ancestors are to be checked.
+        """
+        for ancestor_id in self.ancestors.get(node_id, []):
+            if ancestor_id in self.executed_nodes:
+                # Remove ancestor if all its descendants have been executed
+                if all(descendant in self.executed_nodes for descendant in self.descendants[ancestor_id]):
+                    self._remove_node(ancestor_id)
+
+    def _remove_node(self, node_id):
+        """
+        Remove a node from the cache.
+
+        Args:
+            node_id: The ID of the node to remove.
+        """
+        cache_key = self.cache_key_set.get_data_key(node_id)
+        if cache_key in self.cache:
+            del self.cache[cache_key]
+        subcache_key = self.cache_key_set.get_subcache_key(node_id)
+        if subcache_key in self.subcaches:
+            del self.subcaches[subcache_key]
+
+    def clean_unused(self):
+        """
+        Clean up unused nodes. This is a no-op for this cache implementation.
+        """
+        pass
+
+    def recursive_debug_dump(self):
+        """
+        Dump the cache and dependency graph for debugging.
+
+        Returns:
+            A list containing the cache state and dependency graph.
+        """
+        result = super().recursive_debug_dump()
+        result.append({
+            "descendants": self.descendants,
+            "ancestors": self.ancestors,
+            "executed_nodes": list(self.executed_nodes),
+        })
+        return result

docker-compose.yaml

@@ -0,0 +1,21 @@
+services:
+  comfyui:
+    user: "1000:1000"
+    build: .
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    ports:
+      - "8188:8188"
+    volumes:
+      - "./models:/app/models"
+      - "./input:/app/input"
+      - "./temp:/app/output/temp"
+      - "./output:/app/output"
+      - "./user:/app/user"
+      - "./custom_venv:/app/custom_venv"
+      - "./custom_nodes:/app/custom_nodes"

execution.py

@@ -15,7 +15,7 @@ import nodes
 import comfy.model_management
 from comfy_execution.graph import get_input_info, ExecutionList, DynamicPrompt, ExecutionBlocker
 from comfy_execution.graph_utils import is_link, GraphBuilder
-from comfy_execution.caching import HierarchicalCache, LRUCache, CacheKeySetInputSignature, CacheKeySetID
+from comfy_execution.caching import HierarchicalCache, LRUCache, DependencyAwareCache, CacheKeySetInputSignature, CacheKeySetID
 from comfy_execution.validation import validate_node_input
 
 class ExecutionResult(Enum):
@@ -59,20 +59,27 @@ class IsChangedCache:
             self.is_changed[node_id] = node["is_changed"]
         return self.is_changed[node_id]
 
-class CacheSet:
-    def __init__(self, lru_size=None):
-        if lru_size is None or lru_size == 0:
-            self.init_classic_cache()
-        else:
-            self.init_lru_cache(lru_size)
-        self.all = [self.outputs, self.ui, self.objects]
 
-    # Useful for those with ample RAM/VRAM -- allows experimenting without
+class CacheType(Enum):
-    # blowing away the cache every time
+    CLASSIC = 0
-    def init_lru_cache(self, cache_size):
+    LRU = 1
-        self.outputs = LRUCache(CacheKeySetInputSignature, max_size=cache_size)
+    DEPENDENCY_AWARE = 2
-        self.ui = LRUCache(CacheKeySetInputSignature, max_size=cache_size)
+
-        self.objects = HierarchicalCache(CacheKeySetID)
+
+class CacheSet:
+    def __init__(self, cache_type=None, cache_size=None):
+        if cache_type == CacheType.DEPENDENCY_AWARE:
+            self.init_dependency_aware_cache()
+            logging.info("Disabling intermediate node cache.")
+        elif cache_type == CacheType.LRU:
+            if cache_size is None:
+                cache_size = 0
+            self.init_lru_cache(cache_size)
+            logging.info("Using LRU cache")
+        else:
+            self.init_classic_cache()
+
+        self.all = [self.outputs, self.ui, self.objects]
 
     # Performs like the old cache -- dump data ASAP
     def init_classic_cache(self):
@@ -80,6 +87,17 @@ class CacheSet:
         self.ui = HierarchicalCache(CacheKeySetInputSignature)
         self.objects = HierarchicalCache(CacheKeySetID)
 
+    def init_lru_cache(self, cache_size):
+        self.outputs = LRUCache(CacheKeySetInputSignature, max_size=cache_size)
+        self.ui = LRUCache(CacheKeySetInputSignature, max_size=cache_size)
+        self.objects = HierarchicalCache(CacheKeySetID)
+
+    # only hold cached items while the decendents have not executed
+    def init_dependency_aware_cache(self):
+        self.outputs = DependencyAwareCache(CacheKeySetInputSignature)
+        self.ui = DependencyAwareCache(CacheKeySetInputSignature)
+        self.objects = DependencyAwareCache(CacheKeySetID)
+
     def recursive_debug_dump(self):
         result = {
             "outputs": self.outputs.recursive_debug_dump(),
@@ -414,13 +432,14 @@ def execute(server, dynprompt, caches, current_item, extra_data, executed, promp
     return (ExecutionResult.SUCCESS, None, None)
 
 class PromptExecutor:
-    def __init__(self, server, lru_size=None):
+    def __init__(self, server, cache_type=False, cache_size=None):
-        self.lru_size = lru_size
+        self.cache_size = cache_size
+        self.cache_type = cache_type
         self.server = server
         self.reset()
 
     def reset(self):
-        self.caches = CacheSet(self.lru_size)
+        self.caches = CacheSet(cache_type=self.cache_type, cache_size=self.cache_size)
         self.status_messages = []
         self.success = True
 

main.py

@@ -156,7 +156,13 @@ def cuda_malloc_warning():
 
 def prompt_worker(q, server_instance):
     current_time: float = 0.0
-    e = execution.PromptExecutor(server_instance, lru_size=args.cache_lru)
+    cache_type = execution.CacheType.CLASSIC
+    if args.cache_lru > 0:
+        cache_type = execution.CacheType.LRU
+    elif args.cache_none:
+        cache_type = execution.CacheType.DEPENDENCY_AWARE
+
+    e = execution.PromptExecutor(server_instance, cache_type=cache_type, cache_size=args.cache_lru)
     last_gc_collect = 0
     need_gc = False
     gc_collect_interval = 10.0

requirements.txt

@@ -1,4 +1,4 @@
-comfyui-frontend-package==1.14.6
+comfyui-frontend-package==1.15.13
 torch
 torchsde
 torchvision

server.py

@@ -48,7 +48,7 @@ async def send_socket_catch_exception(function, message):
 @web.middleware
 async def cache_control(request: web.Request, handler):
     response: web.Response = await handler(request)
-    if request.path.endswith('.js') or request.path.endswith('.css'):
+    if request.path.endswith('.js') or request.path.endswith('.css') or request.path.endswith('index.json'):
         response.headers.setdefault('Cache-Control', 'no-cache')
     return response