diff --git a/api_server/routes/internal/internal_routes.py b/api_server/routes/internal/internal_routes.py
index 613b0f7c7..c4dc80c28 100644
--- a/api_server/routes/internal/internal_routes.py
+++ b/api_server/routes/internal/internal_routes.py
@@ -4,6 +4,8 @@ from folder_paths import folder_names_and_paths, get_directory_by_type
 from api_server.services.terminal_service import TerminalService
 import app.logger
 import os
+from utils.extra_config import load_server_config
+
 
 class InternalRoutes:
     '''
@@ -17,6 +19,7 @@ class InternalRoutes:
         self._app: Optional[web.Application] = None
         self.prompt_server = prompt_server
         self.terminal_service = TerminalService(prompt_server)
+        self.config = load_server_config('internal')
 
     def setup_routes(self):
         @self.routes.get('/logs')
@@ -43,7 +46,6 @@ class InternalRoutes:
 
             return web.Response(status=200)
 
-
         @self.routes.get('/folder_paths')
         async def get_folder_paths(request):
             response = {}
@@ -64,6 +66,10 @@ class InternalRoutes:
             )
             return web.json_response([entry.name for entry in sorted_files], status=200)
 
+        @self.routes.get('/models_download/config')
+        async def get_models_download_config(request):
+            response = self.config.get('modelsDownload', {})
+            return web.json_response(response)
 
     def get_app(self):
         if self._app is None:
diff --git a/utils/extra_config.py b/utils/extra_config.py
index a0fcda9e8..303301af0 100644
--- a/utils/extra_config.py
+++ b/utils/extra_config.py
@@ -3,6 +3,28 @@ import yaml
 import folder_paths
 import logging
 
+from .json_util import merge_json_recursive
+
+default_server_config = {
+    'internal': {
+        'modelsDownload': {
+            'allowedSources': [
+                'https://civitai.com/',
+                'https://huggingface.co/'
+            ],
+            'allowedSuffixes': [
+                '.safetensors',
+                '.sft'
+            ],
+            'whitelistedUrls': [
+                'https://huggingface.co/stabilityai/stable-zero123/resolve/main/stable_zero123.ckpt',
+                'https://huggingface.co/TencentARC/T2I-Adapter/resolve/main/models/t2iadapter_depth_sd14v1.pth?download=true',
+                'https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth'
+            ]
+        }
+    }
+}
+
 def load_extra_path_config(yaml_path):
     with open(yaml_path, 'r', encoding='utf-8') as stream:
         config = yaml.safe_load(stream)
@@ -32,3 +54,24 @@ def load_extra_path_config(yaml_path):
                 normalized_path = os.path.normpath(full_path)
                 logging.info("Adding extra search path {} {}".format(x, normalized_path))
                 folder_paths.add_model_folder_path(x, normalized_path, is_default)
+
+def load_server_config(component=None):
+    """
+    Load and returns the server configuration.
+    ensure default configuration is present
+    if a component is specified returns this sub configuration
+
+    Warning: Current merge_json_recursive concatenate arrays and so there is no way to remove default allowed sources for instance
+    """
+    config_path = 'config.yaml'
+    config = dict()
+    try:
+        with open(config_path, 'r', encoding='utf-8') as stream:
+            config = yaml.safe_load(stream)
+    except FileNotFoundError:
+        pass  # Default config could be empty
+
+    config = merge_json_recursive(default_server_config, config)
+    if component is not None:
+        return config.get(component)
+    return config