From 1e343f985352fd6feba80bdc17c4984f7c67cebd Mon Sep 17 00:00:00 2001 From: CaptainB Date: Thu, 30 Jul 2026 15:07:03 +0800 Subject: [PATCH 1/2] feat: integrate Milvus vector store and update PG vector handling --- apps/common/config/embedding_config.py | 9 +- .../migrations/0016_embedding_nullable.py | 21 + apps/knowledge/models/knowledge.py | 2 +- apps/knowledge/serializers/common.py | 70 +- apps/knowledge/sql/blend_text_rank.sql | 5 + apps/knowledge/sql/embedding_recall.sql | 11 + apps/knowledge/tests/fixtures/milvus.sim.yaml | 5 + .../tests/fixtures/milvus/config.yaml | 8 + .../tests/fixtures/milvus/default.js | 74 ++ apps/knowledge/tests/retrieval/README.md | 47 ++ .../tests/retrieval/test_milvus_scope.py | 89 ++ .../tests/retrieval/test_milvus_write.py | 272 ++++++ .../tests/sync/test_workflow_embedding.py | 6 +- apps/knowledge/vector/base_vector.py | 18 + apps/knowledge/vector/milvus_vector.py | 773 ++++++++++++++++++ apps/knowledge/vector/pg_fulltext.py | 100 +++ apps/knowledge/vector/pg_vector.py | 27 +- apps/maxkb/conf.py | 7 + 18 files changed, 1495 insertions(+), 49 deletions(-) create mode 100644 apps/knowledge/migrations/0016_embedding_nullable.py create mode 100644 apps/knowledge/sql/blend_text_rank.sql create mode 100644 apps/knowledge/sql/embedding_recall.sql create mode 100644 apps/knowledge/tests/fixtures/milvus.sim.yaml create mode 100644 apps/knowledge/tests/fixtures/milvus/config.yaml create mode 100644 apps/knowledge/tests/fixtures/milvus/default.js create mode 100644 apps/knowledge/tests/retrieval/README.md create mode 100644 apps/knowledge/tests/retrieval/test_milvus_scope.py create mode 100644 apps/knowledge/tests/retrieval/test_milvus_write.py create mode 100644 apps/knowledge/vector/milvus_vector.py create mode 100644 apps/knowledge/vector/pg_fulltext.py diff --git a/apps/common/config/embedding_config.py b/apps/common/config/embedding_config.py index cff306e737d..522b94bd763 100644 --- a/apps/common/config/embedding_config.py +++ b/apps/common/config/embedding_config.py @@ -90,10 +90,12 @@ def delete_key(_id): class VectorStore: from knowledge.vector.pg_vector import PGVector + from knowledge.vector.milvus_vector import MilvusVector from knowledge.vector.base_vector import BaseVectorStore instance_map = { "pg_vector": PGVector, + "milvus": MilvusVector, } instance = None @@ -104,6 +106,11 @@ def get_embedding_vector() -> BaseVectorStore: if VectorStore.instance is None: from maxkb.const import CONFIG - vector_store_class = VectorStore.instance_map.get(CONFIG.get("VECTOR_STORE_NAME"), PGVector) + # VECTOR_STORE_NAME 显式指定优先；未指定时按 MILVUS_URI 有无自动判断 + # （配了 MILVUS_URI 用 milvus，否则 pg_vector，保证老用户零改动） + name = CONFIG.get("VECTOR_STORE_NAME") + if not name: + name = "milvus" if CONFIG.get("MILVUS_URI") else "pg_vector" + vector_store_class = VectorStore.instance_map.get(name, PGVector) VectorStore.instance = vector_store_class() return VectorStore.instance diff --git a/apps/knowledge/migrations/0016_embedding_nullable.py b/apps/knowledge/migrations/0016_embedding_nullable.py new file mode 100644 index 00000000000..a32b9a35021 --- /dev/null +++ b/apps/knowledge/migrations/0016_embedding_nullable.py @@ -0,0 +1,21 @@ +# Generated for Milvus vector store support + +from django.db import migrations + +import knowledge.models.knowledge + + +class Migration(migrations.Migration): + dependencies = [ + ("knowledge", "0015_knowledge_external_service"), + ] + + operations = [ + # Milvus 混合存储模式下，向量交由 Milvus 保存，PG 的 embedding 列允许为空。 + # pg_vector 模式不受影响（仍写入真实向量）。 + migrations.AlterField( + model_name="embedding", + name="embedding", + field=knowledge.models.knowledge.VectorField(null=True, verbose_name="向量"), + ), + ] diff --git a/apps/knowledge/models/knowledge.py b/apps/knowledge/models/knowledge.py index 848260b889d..3569e52e898 100644 --- a/apps/knowledge/models/knowledge.py +++ b/apps/knowledge/models/knowledge.py @@ -560,7 +560,7 @@ class Embedding(models.Model): knowledge = models.ForeignKey(Knowledge, on_delete=models.DO_NOTHING, verbose_name="文档关联", db_constraint=False) document = models.ForeignKey(Document, on_delete=models.DO_NOTHING, verbose_name="文档关联", db_constraint=False) paragraph = models.ForeignKey(Paragraph, on_delete=models.DO_NOTHING, verbose_name="段落关联", db_constraint=False) - embedding = VectorField(verbose_name="向量") + embedding = VectorField(verbose_name="向量", null=True) search_vector = SearchVectorField(verbose_name="分词", default="") meta = models.JSONField(verbose_name="元数据", default=dict) diff --git a/apps/knowledge/serializers/common.py b/apps/knowledge/serializers/common.py index e628abb4e0b..95e7f42a0be 100644 --- a/apps/knowledge/serializers/common.py +++ b/apps/knowledge/serializers/common.py @@ -13,29 +13,36 @@ from typing import List import uuid_utils.compat as uuid -from django.db.models import QuerySet -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from system_manage.services.resource_mapping import ( - save_workflow_mapping, - get_instance_resource, - knowledge_instance_field_call_dict, -) -from common.config.embedding_config import ModelManage +from common.config.embedding_config import ModelManage, VectorStore from common.db.search import native_search -from common.db.sql_execute import sql_execute, update_execute +from common.db.sql_execute import update_execute from common.exception.app_exception import AppApiException from common.utils.common import get_file_content from common.utils.fork import Fork -from common.utils.logger import maxkb_logger -from knowledge.models import Document, DocumentResourceType, KnowledgeWorkflow, KnowledgeType -from knowledge.models import Paragraph, Problem, ProblemParagraphMapping, Knowledge, File +from django.db.models import QuerySet +from django.utils.translation import gettext_lazy as _ +from knowledge.models import ( + Document, + DocumentResourceType, + File, + Knowledge, + KnowledgeType, + KnowledgeWorkflow, + Paragraph, + Problem, + ProblemParagraphMapping, +) from knowledge.serializers.document_strategy import DocumentStrategySerializer from knowledge.services.multimodal_retrieval import enrich_paragraph_sources from maxkb.conf import PROJECT_DIR from models_provider.tools import get_model, get_model_default_params +from rest_framework import serializers from system_manage.models.resource_mapping import ResourceType +from system_manage.services.resource_mapping import ( + get_instance_resource, + knowledge_instance_field_call_dict, + save_workflow_mapping, +) class MetaSerializer(serializers.Serializer): @@ -281,45 +288,22 @@ def create_knowledge_index(knowledge_id=None, document_id=None): if knowledge_id is None and document_id is None: raise AppApiException(500, _("Knowledge ID or Document ID must be provided")) - if knowledge_id is not None: - k_id = knowledge_id - else: + if knowledge_id is None: document = QuerySet(Document).filter(id=document_id).first() if document is None: return - k_id = document.knowledge_id - - sql = f"SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'embedding' AND indexname = 'embedding_hnsw_idx_{k_id}'" - index = sql_execute(sql, []) - if not index: - sql = f"SELECT vector_dims(embedding) AS dims FROM embedding WHERE knowledge_id = '{k_id}' LIMIT 1" - result = sql_execute(sql, []) - if len(result) == 0: - return - dims = result[0]["dims"] - # 超过2000维度不创建索引，pgvector hnsw索引不支持超过2000维度 - if dims < 2000: - sql = f"""CREATE INDEX "embedding_hnsw_idx_{k_id}" ON embedding USING hnsw ((embedding::vector({dims})) vector_cosine_ops) WHERE knowledge_id = '{k_id}'""" - update_execute(sql, []) - maxkb_logger.info(f"Created index for knowledge ID: {k_id}") + knowledge_id = document.knowledge_id + VectorStore.get_embedding_vector().ensure_knowledge_index(knowledge_id) def drop_knowledge_index(knowledge_id=None, document_id=None): if knowledge_id is None and document_id is None: raise AppApiException(500, _("Knowledge ID or Document ID must be provided")) - if knowledge_id is not None: - k_id = knowledge_id - else: + if knowledge_id is None: document = QuerySet(Document).filter(id=document_id).first() - k_id = document.knowledge_id - - sql = f"SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'embedding' AND indexname = 'embedding_hnsw_idx_{k_id}'" - index = sql_execute(sql, []) - if index: - sql = f'DROP INDEX "embedding_hnsw_idx_{k_id}"' - update_execute(sql, []) - maxkb_logger.info(f"Dropped index for knowledge ID: {k_id}") + knowledge_id = document.knowledge_id + VectorStore.get_embedding_vector().drop_knowledge_index(knowledge_id) def update_resource_mapping_by_knowledge(knowledge_id: str): diff --git a/apps/knowledge/sql/blend_text_rank.sql b/apps/knowledge/sql/blend_text_rank.sql new file mode 100644 index 00000000000..fbbdd4c5545 --- /dev/null +++ b/apps/knowledge/sql/blend_text_rank.sql @@ -0,0 +1,5 @@ +SELECT + id, + paragraph_id, + ts_rank_cd(search_vector, websearch_to_tsquery('simple', %s), 32) AS text_rank +FROM embedding ${embedding_query} diff --git a/apps/knowledge/sql/embedding_recall.sql b/apps/knowledge/sql/embedding_recall.sql new file mode 100644 index 00000000000..256ceff893f --- /dev/null +++ b/apps/knowledge/sql/embedding_recall.sql @@ -0,0 +1,11 @@ +SELECT + id, + paragraph_id, + source_id, + source_type, + meta, + (1 - (embedding::vector(%s) <=> %s)) AS similarity +FROM embedding ${embedding_query} +AND embedding IS NOT NULL +ORDER BY (embedding::vector(%s) <=> %s) +LIMIT LEAST(%s * 10, 500) diff --git a/apps/knowledge/tests/fixtures/milvus.sim.yaml b/apps/knowledge/tests/fixtures/milvus.sim.yaml new file mode 100644 index 00000000000..891c5a1c514 --- /dev/null +++ b/apps/knowledge/tests/fixtures/milvus.sim.yaml @@ -0,0 +1,5 @@ +simulation: milvus-write-regression +routing: path +mocks: + - routeKey: milvus + path: ./milvus/config.yaml diff --git a/apps/knowledge/tests/fixtures/milvus/config.yaml b/apps/knowledge/tests/fixtures/milvus/config.yaml new file mode 100644 index 00000000000..2b00244ac43 --- /dev/null +++ b/apps/knowledge/tests/fixtures/milvus/config.yaml @@ -0,0 +1,8 @@ +version: 1 +name: milvus-write-regression +protocol: http +interceptRequests: true +scenarios: + - name: default + path: ./default.js + default: true diff --git a/apps/knowledge/tests/fixtures/milvus/default.js b/apps/knowledge/tests/fixtures/milvus/default.js new file mode 100644 index 00000000000..a0c914bfc40 --- /dev/null +++ b/apps/knowledge/tests/fixtures/milvus/default.js @@ -0,0 +1,74 @@ +// Contract: https://milvus.io/api-reference/restful/v2.6.x/v2/Collection%20(v2)/Create.md +// JSON field names: https://github.com/milvus-io/milvus/blob/v2.6.0/internal/distributed/proxy/httpserver/request_v2.go +// Synthetic state and failure controls exercise the real Python HTTP client, not vector search quality. +const http = require("node:http"); +const prefix = "/v2/vectordb/"; + +http.createServer(async (request, response) => { + const path = new URL(request.url, "http://localhost").pathname; + const send = (status, body, raw = false) => { + response.writeHead(status, { "Content-Type": raw ? "text/html" : "application/json" }); + response.end(raw ? body : JSON.stringify(body)); + }; + try { + if (path === "/__test/reset" && request.method === "POST") { + await pm.state.set("milvus:state", { collections: {}, calls: [], failure: null, ...request.body }); + return send(200, { code: 0 }); + } + const state = await pm.state.get("milvus:state"); + if (path === "/__test/state" && request.method === "GET") return send(200, state); + if (request.method !== "POST" || !path.startsWith(prefix)) return send(501, { message: "Endpoint not modeled" }); + if (request.headers.authorization !== "Bearer regression-token") return send(401, { code: 1800, message: "Unauthorized" }); + const body = request.body; + const operation = path.slice(prefix.length); + state.calls.push({ operation, body }); + await pm.state.set("milvus:state", state); + const failure = state.failure; + if (failure?.operation === operation && state.calls.filter(call => call.operation === operation).length === (failure.at || 1)) { + if (failure.disconnect) return request.socket.destroy(); + return send(failure.status || 200, failure.body, failure.raw); + } + const collection = state.collections[body.collectionName]; + const fail = message => send(200, { code: 1100, message }); + if (operation === "collections/has") return send(200, { code: 0, data: { has: !!collection } }); + if (operation === "collections/create") { + const schema = body.schema; + if (!schema || schema.autoID !== false || schema.enableDynamicField !== false) return fail("Invalid collection schema"); + const fields = schema.fields || []; + const primary = fields.find(field => field.isPrimary); + const vector = fields.find(field => field.dataType === "FloatVector"); + if (primary?.fieldName !== "id" || primary.dataType !== "VarChar") return fail("Expected VarChar primary key"); + if (!vector || !Number.isInteger(Number(vector.elementTypeParams?.dim)) || Number(vector.elementTypeParams.dim) <= 0) { + return fail("FloatVector requires elementTypeParams.dim"); + } + for (const field of fields) { + if ("dim" in field || "maxLength" in field) return fail("Field parameters must be nested in elementTypeParams"); + if (field.dataType === "VarChar" && !(Number(field.elementTypeParams?.max_length) > 0)) { + return fail("VarChar requires elementTypeParams.max_length"); + } + } + if (collection) return fail("Collection already exists"); + state.collections[body.collectionName] = { schema, dim: Number(vector.elementTypeParams.dim), loaded: false, records: {} }; + } else if (operation === "collections/load") { + if (!collection) return fail("Collection not found"); + collection.loaded = true; + } else if (operation === "entities/upsert") { + if (!collection) return fail("Collection not found"); + if (!Array.isArray(body.data) || !body.data.length) return fail("Missing entities"); + for (const record of body.data) { + if (typeof record.id !== "string" || record.embedding?.length !== collection.dim) return fail("Entity schema mismatch"); + collection.records[record.id] = record; + } + await pm.state.set("milvus:state", state); + return send(200, { code: 0, data: { upsertCount: body.data.length, upsertIds: body.data.map(record => record.id) } }); + } else if (operation === "collections/drop") { + delete state.collections[body.collectionName]; + } else { + return send(501, { message: "Endpoint not modeled" }); + } + await pm.state.set("milvus:state", state); + return send(200, { code: 0, data: {} }); + } catch (error) { + return send(500, { code: 65535, message: error.message }); + } +}).listen(process.env.PORT, "127.0.0.1"); diff --git a/apps/knowledge/tests/retrieval/README.md b/apps/knowledge/tests/retrieval/README.md new file mode 100644 index 00000000000..07cb934fed0 --- /dev/null +++ b/apps/knowledge/tests/retrieval/README.md @@ -0,0 +1,47 @@ +# 检索回归测试 + +常规检索测试： + +```bash +python apps/manage.py test knowledge.tests.retrieval +``` + +## Milvus 写入 HTTP 回归 + +使用 Postman CLI 启动仅本机的模拟接口（本次验证版本为 1.74.0）： + +```bash +postman simulation run apps/knowledge/tests/fixtures/milvus.sim.yaml --port 49644 --no-history +``` + +另一个终端运行真实 Python HTTP 客户端的回归测试： + +```bash +MILVUS_TEST_URL=http://127.0.0.1:49644/milvus \ + python apps/manage.py test knowledge.tests.retrieval.test_milvus_write +``` + +未设置 `MILVUS_TEST_URL` 时仍运行建表参数、写入失败上抛的核心单元回归，仅跳过 HTTP 测试。 +该地址必须指向本地测试模拟接口： +测试会调用模拟服务专用的 `/__test/reset`、`/__test/state` 控制接口。 +测试隔离并恢复 Milvus 配置、Session 与 collection 缓存，使用虚拟凭据和合成向量。 +PostgreSQL 查询与模型推理使用测试替身，不连接真实数据库或模型服务。 + +接口契约来源： + +- [Milvus REST v2 创建 collection](https://milvus.io/api-reference/restful/v2.6.x/v2/Collection%20%28v2%29/Create.md) +- [Milvus REST v2 upsert](https://milvus.io/api-reference/restful/v2.6.x/v2/Vector%20%28v2%29/Upsert.md) +- [Milvus 2.6 服务端 JSON 字段定义](https://github.com/milvus-io/milvus/blob/v2.6.0/internal/distributed/proxy/httpserver/request_v2.go) + +模拟服务依据上述公开契约编写，未使用提供方的托管 mock。 +它检查 `elementTypeParams.dim`、所有 VarChar 字段的 `elementTypeParams.max_length`、主键和实体维度。 +测试覆盖创建、加载、写入、加载失败后的重试、删除后重建、批次中途失败、HTTP/业务错误、 +无效 JSON、缺失或类型错误的业务码、连接失败，以及段落向量化失败状态和读取时的 PG 兜底。 + +写入路径要求 HTTP 2xx 且业务 `code` 为整数 `0`；检查、创建、加载或 upsert 失败时抛出 +`AppApiException`，阻止后续批次并由调用方更新向量化状态。加载成功与 collection 存在分别缓存， +避免加载失败后的重试被存在性缓存跳过。 + +这些测试验证客户端与本地 HTTP 契约，不验证真实 Milvus 的索引与检索效果。 +异常上抛不提供 PostgreSQL 与 Milvus 的分布式事务：此前成功的批次和已写入的 PG 元数据可能保留； +已有段落重新向量化流程会先清理该段落的旧向量，再重新写入。 diff --git a/apps/knowledge/tests/retrieval/test_milvus_scope.py b/apps/knowledge/tests/retrieval/test_milvus_scope.py new file mode 100644 index 00000000000..c43d2bd09fa --- /dev/null +++ b/apps/knowledge/tests/retrieval/test_milvus_scope.py @@ -0,0 +1,89 @@ +from unittest.mock import MagicMock, call, patch +from uuid import UUID + +from django.test import SimpleTestCase + +from knowledge.models import SearchMode +from knowledge.vector.milvus_vector import MilvusVector + + +class MilvusWorkflowImageScopeTests(SimpleTestCase): + @patch("knowledge.vector.milvus_vector.text_rank_by_ids", return_value={}) + @patch("knowledge.vector.milvus_vector.pg_vector_recall", return_value=[]) + @patch.object(MilvusVector, "_milvus_search") + def test_text_and_images_keep_filters_in_milvus_and_pg_fallback(self, search, pg_recall, _text_rank): + search.side_effect = [ + [{"id": "text", "paragraph_id": "same-paragraph", "similarity": 0.7}], + [{"id": "image-1", "paragraph_id": "same-paragraph", "similarity": 0.9}], + [{"id": "image-2", "paragraph_id": "other-paragraph", "similarity": 0.8}], + ] + model = MagicMock() + model.embed_query.return_value = [1.0, 0.0] + model.embed_images.return_value = [[0.0, 1.0], [0.5, 0.5]] + + result = MilvusVector().hit_test( + "chart", + ["knowledge"], + ["disabled-document"], + 2, + 0.6, + SearchMode.blend, + model, + ["image-1", "image-2"], + document_id_list=["selected-document"], + exclude_paragraph_id_list=["previous-hit"], + ) + + self.assertEqual([hit["paragraph_id"] for hit in result], ["same-paragraph", "other-paragraph"]) + self.assertEqual(result[0]["query_unit_type"], "image") + self.assertEqual(result[0]["query_unit_index"], 1) + embeddings = [[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]] + self.assertEqual( + search.call_args_list, + [ + call("knowledge", vector, 2, True, ["selected-document"], ["disabled-document"], ["previous-hit"]) + for vector in embeddings + ], + ) + self.assertEqual( + pg_recall.call_args_list, + [ + call(vector, ["knowledge"], True, ["selected-document"], ["disabled-document"], ["previous-hit"], 2) + for vector in embeddings + ], + ) + + @patch("knowledge.vector.milvus_vector.KeywordsSearch.handle", return_value=[]) + def test_keyword_search_keeps_document_and_paragraph_filters(self, search): + knowledge_id, disabled_document, selected_document, previous_hit = [UUID(int=index) for index in range(1, 5)] + + MilvusVector().hit_test( + "chart", + [str(knowledge_id)], + [str(disabled_document)], + 2, + 0.6, + SearchMode.keywords, + MagicMock(), + document_id_list=[str(selected_document)], + exclude_paragraph_id_list=[str(previous_hit)], + ) + + search.assert_called_once() + sql, parameters = search.call_args.args[0].query.sql_with_params() + self.assertEqual(sql.count("NOT ("), 2) + for identifier in (knowledge_id, disabled_document, selected_document, previous_hit): + self.assertIn(identifier, parameters) + + @patch.object(MilvusVector, "_dispatch_search") + def test_empty_document_scope_skips_embedding_and_search(self, search): + model = MagicMock() + for mode in (SearchMode.embedding, SearchMode.blend, SearchMode.keywords): + with self.subTest(mode=mode): + result = MilvusVector().hit_test( + "chart", ["knowledge"], [], 3, 0.6, mode, model, ["image"], document_id_list=[] + ) + self.assertEqual(result, []) + + self.assertEqual(model.mock_calls, []) + search.assert_not_called() diff --git a/apps/knowledge/tests/retrieval/test_milvus_write.py b/apps/knowledge/tests/retrieval/test_milvus_write.py new file mode 100644 index 00000000000..17c80d35543 --- /dev/null +++ b/apps/knowledge/tests/retrieval/test_milvus_write.py @@ -0,0 +1,272 @@ +"""Milvus REST v2 contract and failure propagation through the real HTTP client.""" + +import os +import socket +from types import SimpleNamespace +from unittest import skipUnless +from unittest.mock import MagicMock, patch +from uuid import UUID + +import requests +from django.test import SimpleTestCase + +from common.event.listener_manage import ListenerManagement +from common.exception.app_exception import AppApiException +from knowledge.models import SourceType, State, Status, TaskType +from knowledge.vector.milvus_vector import MilvusVector +from maxkb.const import CONFIG + + +class MilvusWriteTests(SimpleTestCase): + """Core regressions also run without the optional HTTP simulation.""" + + def setUp(self): + self.enterContext(patch.object(MilvusVector, "_existing", set())) + self.enterContext(patch.object(MilvusVector, "_loaded", set())) + self.enterContext(patch("knowledge.vector.milvus_vector.maxkb_logger")) + self.post = self.enterContext(patch.object(MilvusVector, "_post")) + self.store = MilvusVector() + self.responses = [ + (True, {"code": 0, "data": {"has": False}}), + (True, {"code": 0}), + (True, {"code": 0}), + (True, {"code": 0}), + ] + + def test_collection_field_parameters_follow_rest_v2_schema(self): + self.post.side_effect = self.responses + self.store._upsert_records("knowledge", [{"id": "entity", "embedding": [1.0, 0.0]}]) + + create_body = next( + call.args[1] for call in self.post.call_args_list if call.args[0].endswith("/collections/create") + ) + fields = {field["fieldName"]: field for field in create_body["schema"]["fields"]} + self.assertEqual(fields["embedding"]["elementTypeParams"], {"dim": 2}) + for field in fields.values(): + if field["dataType"] == "VarChar": + self.assertEqual(field["elementTypeParams"], {"max_length": 128}) + self.assertNotIn("dim", field) + self.assertNotIn("maxLength", field) + + def test_write_failure_raises_and_stops_subsequent_requests(self): + operations = ["collections/has", "collections/create", "collections/load", "entities/upsert"] + for index, operation in enumerate(operations): + with self.subTest(operation=operation): + MilvusVector._existing.clear() + MilvusVector._loaded.clear() + self.post.reset_mock() + self.post.side_effect = self.responses[:index] + [(False, {"code": 65535, "message": "rejected"})] + + with self.assertRaises(AppApiException) as error: + self.store._upsert_records("knowledge", [{"id": "entity", "embedding": [1.0, 0.0]}]) + + self.assertIn(operation, error.exception.message) + self.assertEqual(self.post.call_count, index + 1) + + +@skipUnless(os.environ.get("MILVUS_TEST_URL"), "Start the local Milvus Postman simulation") +class MilvusWriteHTTPTests(SimpleTestCase): + knowledge_id = "00000000-0000-0000-0000-000000000001" + + def setUp(self): + self.base_url = os.environ["MILVUS_TEST_URL"].rstrip("/") + self.enterContext(patch.dict(CONFIG, MILVUS_URI=self.base_url, MILVUS_TOKEN="regression-token")) + self.enterContext(patch.object(MilvusVector, "_session", None)) + self.enterContext(patch.object(MilvusVector, "_uri", "")) + self.enterContext(patch.object(MilvusVector, "_existing", set())) + self.enterContext(patch.object(MilvusVector, "_loaded", set())) + self.enterContext(patch("knowledge.vector.milvus_vector.maxkb_logger")) + self.addCleanup(self.close_session) + self.store = MilvusVector() + self.name = self.store._collection_name(self.knowledge_id) + self.reset() + + def close_session(self): + if MilvusVector._session is not None: + MilvusVector._session.close() + + def reset(self, failure=None, existing=False): + MilvusVector._existing.clear() + MilvusVector._loaded.clear() + collections = {self.name: {"dim": 2, "loaded": False, "records": {}}} if existing else {} + response = requests.post( + self.base_url + "/__test/reset", json={"failure": failure, "collections": collections}, timeout=5 + ) + response.raise_for_status() + + def state(self): + response = requests.get(self.base_url + "/__test/state", timeout=5) + response.raise_for_status() + return response.json() + + def records(self, count=1): + return [ + { + "id": str(index), + "embedding": [1.0, 0.0], + "document_id": "document", + "paragraph_id": "paragraph", + "is_active": True, + "source_id": "paragraph", + "source_type": SourceType.PARAGRAPH.value, + } + for index in range(count) + ] + + def test_custom_schema_creates_loads_and_upserts_entities(self): + self.store._upsert_records(self.knowledge_id, self.records()) + + state = self.state() + self.assertEqual( + [call["operation"] for call in state["calls"]], + [ + "collections/has", + "collections/create", + "collections/load", + "entities/upsert", + ], + ) + collection = state["collections"][self.name] + self.assertTrue(collection["loaded"]) + self.assertEqual(collection["records"]["0"], self.records()[0]) + fields = {field["fieldName"]: field for field in collection["schema"]["fields"]} + self.assertEqual(int(fields["embedding"]["elementTypeParams"]["dim"]), 2) + for field in fields.values(): + if field["dataType"] == "VarChar": + self.assertEqual(int(field["elementTypeParams"]["max_length"]), 128) + self.assertEqual(state["calls"][1]["body"]["indexParams"][0]["indexType"], "HNSW") + + def test_collection_failures_stop_before_upsert(self): + operations = ["collections/has", "collections/create", "collections/load"] + for index, operation in enumerate(operations): + with self.subTest(operation=operation): + self.reset({"operation": operation, "body": {"code": 65535, "message": "collection unavailable"}}) + with self.assertRaises(AppApiException) as error: + self.store._upsert_records(self.knowledge_id, self.records()) + self.assertIn(operation, error.exception.message) + self.assertEqual([call["operation"] for call in self.state()["calls"]], operations[: index + 1]) + + def test_failed_load_is_retried_for_an_existing_collection(self): + self.reset({"operation": "collections/load", "body": {"code": 65535, "message": "loading failed"}}) + with self.assertRaises(AppApiException): + self.store._upsert_records(self.knowledge_id, self.records()) + + self.store._upsert_records(self.knowledge_id, self.records()) + + state = self.state() + operations = [call["operation"] for call in state["calls"]] + self.assertEqual(operations.count("collections/create"), 1) + self.assertEqual(operations.count("collections/load"), 2) + self.assertTrue(state["collections"][self.name]["loaded"]) + self.assertEqual(len(state["collections"][self.name]["records"]), 1) + + def test_upsert_rejects_http_business_and_malformed_responses(self): + cases = [ + (200, {"code": 65535, "message": "write rejected"}, False), + (401, {"message": "Unauthorized"}, False), + (429, {"code": 0}, False), + (503, {"code": 0}, False), + (502, "Bad gateway", True), + (200, "not JSON", True), + (200, {}, False), + (200, [], False), + (200, None, False), + (200, {"code": False}, False), + (200, {"code": "0"}, False), + ] + for status, body, raw in cases: + with self.subTest(status=status, body=body): + self.reset({"operation": "entities/upsert", "status": status, "body": body, "raw": raw}, existing=True) + with self.assertRaises(AppApiException) as error: + self.store._upsert_records(self.knowledge_id, self.records()) + self.assertIn("entities/upsert", error.exception.message) + + def test_connection_loss_is_propagated(self): + self.reset({"operation": "entities/upsert", "disconnect": True}, existing=True) + with self.assertRaises(AppApiException): + self.store._upsert_records(self.knowledge_id, self.records()) + + def test_unreachable_service_is_propagated(self): + # Reserve an unused local port without listening; no external service is contacted. + with socket.socket() as unavailable: + unavailable.bind(("127.0.0.1", 0)) + uri = f"http://127.0.0.1:{unavailable.getsockname()[1]}" + with patch.dict(CONFIG, MILVUS_URI=uri), self.assertRaises(AppApiException) as error: + self.store._upsert_records(self.knowledge_id, self.records()) + self.assertIn("collections/has", error.exception.message) + + def test_malformed_has_result_does_not_trigger_create(self): + for payload in (None, [], {}, {"has": "false"}): + with self.subTest(payload=payload): + self.reset({"operation": "collections/has", "body": {"code": 0, "data": payload}}) + with self.assertRaises(AppApiException): + self.store._upsert_records(self.knowledge_id, self.records()) + self.assertEqual([call["operation"] for call in self.state()["calls"]], ["collections/has"]) + + def test_drop_invalidates_loaded_cache_for_uuid_and_string_ids(self): + self.store._upsert_records(self.knowledge_id, self.records()) + self.store.drop_knowledge_index(UUID(self.knowledge_id)) + self.store._upsert_records(self.knowledge_id, self.records()) + operations = [call["operation"] for call in self.state()["calls"]] + self.assertEqual(operations.count("collections/create"), 2) + self.assertEqual(operations.count("collections/load"), 2) + + @patch("knowledge.vector.milvus_vector.pg_vector_recall") + def test_read_failure_keeps_pg_fallback(self, pg_recall): + self.reset({"operation": "collections/has", "status": 503, "body": {"message": "unavailable"}}) + pg_recall.return_value = [{"paragraph_id": "paragraph", "similarity": 0.9}] + result = self.store._search_embedding([1.0, 0.0], [self.knowledge_id], None, [], [], True, 3, 0.5) + self.assertEqual([row["paragraph_id"] for row in result], ["paragraph"]) + pg_recall.assert_called_once() + + def test_later_upsert_failure_stops_remaining_batches(self): + self.reset( + {"operation": "entities/upsert", "at": 2, "body": {"code": 65535, "message": "write rejected"}}, + existing=True, + ) + with self.assertRaises(AppApiException): + self.store._upsert_records(self.knowledge_id, self.records(1001)) + + state = self.state() + batches = [call for call in state["calls"] if call["operation"] == "entities/upsert"] + self.assertEqual([len(call["body"]["data"]) for call in batches], [500, 500]) + self.assertEqual(len(state["collections"][self.name]["records"]), 500) + + def test_paragraph_embedding_failure_is_not_reported_as_success(self): + self.reset( + {"operation": "entities/upsert", "body": {"code": 65535, "message": "write rejected"}}, existing=True + ) + model = MagicMock() + model.embed_documents.side_effect = lambda texts: [[1.0, 0.0] for _ in texts] + data = [ + { + "text": "paragraph text", + "knowledge_id": self.knowledge_id, + "document_id": "document", + "paragraph_id": "paragraph", + "source_id": "paragraph", + "source_type": SourceType.PARAGRAPH.value, + } + ] + with ( + patch("common.event.listener_manage.VectorStore.get_embedding_vector", return_value=self.store), + patch("common.event.listener_manage.process_pending_visual_assets"), + patch("common.event.listener_manage.embed_paragraph_assets") as embed_assets, + patch("common.event.listener_manage.native_search", return_value=data), + patch("common.event.listener_manage.QuerySet") as listener_query, + patch("knowledge.vector.milvus_vector.QuerySet") as vector_query, + patch.object(ListenerManagement, "update_status") as update_status, + patch("common.event.listener_manage.maxkb_logger") as logger, + ): + listener_query.return_value.filter.return_value.first.return_value = SimpleNamespace(status=str(Status())) + vector_query.return_value.filter.return_value.values.return_value = [] + vector_query.return_value.filter.return_value.values_list.return_value = [] + + ListenerManagement.embedding_by_paragraph("paragraph", model) + + self.assertEqual([call.args[2] for call in update_status.call_args_list], [State.STARTED, State.FAILURE]) + self.assertTrue(all(call.args[1] == TaskType.EMBEDDING for call in update_status.call_args_list)) + embed_assets.assert_not_called() + self.assertTrue( + any(call["operation"] == "entities/upsert" for call in self.state()["calls"]), logger.error.call_args + ) diff --git a/apps/knowledge/tests/sync/test_workflow_embedding.py b/apps/knowledge/tests/sync/test_workflow_embedding.py index d79004c55d5..4f645884ac3 100644 --- a/apps/knowledge/tests/sync/test_workflow_embedding.py +++ b/apps/knowledge/tests/sync/test_workflow_embedding.py @@ -109,9 +109,9 @@ def test_lock_is_released_even_when_status_finalization_fails(self): ListenerManagement.embedding_by_document("deleted", MagicMock()) self.lock.un_lock.assert_called_once_with("embedding:deleted") - @patch("knowledge.serializers.common.sql_execute") + @patch("knowledge.serializers.common.VectorStore.get_embedding_vector") @patch("knowledge.serializers.common.QuerySet") - def test_document_deleted_before_index_lookup_skips_index_sql(self, query_set, execute_sql): + def test_document_deleted_before_index_lookup_skips_vector_store(self, query_set, get_vector_store): query_set.return_value.filter.return_value.first.return_value = None create_knowledge_index(document_id="deleted") - execute_sql.assert_not_called() + get_vector_store.assert_not_called() diff --git a/apps/knowledge/vector/base_vector.py b/apps/knowledge/vector/base_vector.py index ee4c88f1144..1c105b40821 100644 --- a/apps/knowledge/vector/base_vector.py +++ b/apps/knowledge/vector/base_vector.py @@ -74,6 +74,24 @@ def vector_create(self): """ pass + def ensure_knowledge_index(self, knowledge_id: str, dim: int = None): + """ + 创建知识库向量索引。默认空实现，由具体向量库按需覆盖 + （PGVector 建 HNSW，Milvus 建 collection）。 + :param knowledge_id: 知识库id + :param dim: 向量维度（部分向量库建索引时需要） + :return: + """ + return None + + def drop_knowledge_index(self, knowledge_id: str): + """ + 删除知识库向量索引。默认空实现。 + :param knowledge_id: 知识库id + :return: + """ + return None + def save_pre_handler(self): """ 插入前置处理器 主要是判断向量库是否创建 diff --git a/apps/knowledge/vector/milvus_vector.py b/apps/knowledge/vector/milvus_vector.py new file mode 100644 index 00000000000..790ad60be9f --- /dev/null +++ b/apps/knowledge/vector/milvus_vector.py @@ -0,0 +1,773 @@ +# coding=utf-8 +""" +@project: maxkb +@file： milvus_vector.py +@desc: Milvus 向量库实现（REST API 接入，不依赖 pymilvus SDK）。混合存储： + 向量存 Milvus，元数据 + tsvector 全文存 PG。每个知识库一个 collection + （maxkb_kb_{knowledge_id}），HNSW + COSINE。 + PG 的 Embedding 表为元数据 source of truth，Milvus 按 Embedding.id 镜像向量。 + 连接通过 MILVUS_URI + MILVUS_TOKEN（Authorization: Bearer {token}）。 +""" + +from collections import defaultdict +from typing import Dict, List + +import requests +import uuid_utils.compat as uuid +from common.exception.app_exception import AppApiException +from common.utils.logger import maxkb_logger +from common.utils.ts_vecto_util import to_ts_vector +from django.contrib.postgres.search import SearchVector +from django.db.models import QuerySet, Value +from django.utils.translation import gettext_lazy as _ +from knowledge.models import Embedding, SearchMode, Termbase +from knowledge.vector.base_vector import BaseVectorStore, normalize_for_embedding +from knowledge.vector.pg_fulltext import text_rank_by_ids, pg_vector_recall +from knowledge.vector.pg_vector import KeywordsSearch +from models_provider.base_model_provider import MaxKBBaseEmbeddingModel + +# Milvus REST API v2 路径 +_PATH_HAS = "/v2/vectordb/collections/has" +_PATH_CREATE = "/v2/vectordb/collections/create" +_PATH_DROP = "/v2/vectordb/collections/drop" +_PATH_LOAD = "/v2/vectordb/collections/load" +_PATH_UPSERT = "/v2/vectordb/entities/upsert" +_PATH_DELETE = "/v2/vectordb/entities/delete" +_PATH_SEARCH = "/v2/vectordb/entities/search" +_PATH_QUERY = "/v2/vectordb/entities/query" + + +class MilvusVector(BaseVectorStore): + _session = None + _uri = "" + _existing = set() # 已确认存在的 collection 缓存（drop 时失效） + _loaded = set() # 本进程确认加载成功后才允许跳过写入前准备 + + # ------------------------------------------------------------------ # + # HTTP 封装 + # ------------------------------------------------------------------ # + @classmethod + def _get_session(cls): + if cls._session is None: + from maxkb.const import CONFIG + + # MILVUS_URI 为空时回退本机（仅显式选 milvus 但未配 URI 时触发） + cls._uri = (CONFIG.get("MILVUS_URI") or "http://127.0.0.1:19530").rstrip("/") + token = CONFIG.get("MILVUS_TOKEN", "") or "" + s = requests.Session() + s.headers.update({"Content-Type": "application/json"}) + if token: + s.headers.update({"Authorization": f"Bearer {token}"}) + cls._session = s + return cls._session + + @classmethod + def _post(cls, path: str, body: dict): + """调用 Milvus REST，仅 HTTP 2xx 且业务 code 为整数 0 时成功。""" + s = cls._get_session() + url = f"{cls._uri}{path}" + try: + resp = s.post(url, json=body, timeout=120) + except requests.RequestException as e: + maxkb_logger.error(f"Milvus REST {path} request error: {e}") + return False, {"code": -1, "message": str(e)} + try: + data = resp.json() + except ValueError: + maxkb_logger.error(f"Milvus REST {path} bad response: {resp.status_code} {resp.text[:200]}") + return False, {"code": -1, "message": f"HTTP {resp.status_code}: invalid JSON response"} + if not 200 <= resp.status_code < 300: + message = data.get("message", resp.reason) if isinstance(data, dict) else resp.reason + return False, {"code": resp.status_code, "message": f"HTTP {resp.status_code}: {message}"} + if not isinstance(data, dict) or type(data.get("code")) is not int: + return False, {"code": -1, "message": "Invalid Milvus response: expected an integer code"} + if data["code"] != 0: + return False, data + return True, data + + @classmethod + def _post_required(cls, path: str, body: dict): + """写入依赖的请求必须成功，异常交由调用方更新向量化失败状态。""" + ok, data = cls._post(path, body) + if not ok: + message = ( + f"Milvus request failed ({path}, collection={body.get('collectionName')}, " + f"code={data.get('code')}): {data.get('message', 'Unknown error')}" + ) + maxkb_logger.error(message) + raise AppApiException(500, message) + return data + + @staticmethod + def _collection_name(knowledge_id: str) -> str: + # Milvus collection 名仅允许字母/数字/下划线，uuid 含 '-' 需替换 + return f"maxkb_kb_{str(knowledge_id).replace('-', '_')}" + + def _has_collection(self, knowledge_id: str, *, raise_on_error=False) -> bool: + knowledge_id = str(knowledge_id) + if knowledge_id in MilvusVector._existing: + return True + body = {"collectionName": self._collection_name(knowledge_id)} + if raise_on_error: + data = self._post_required(_PATH_HAS, body) + ok = True + else: + ok, data = self._post(_PATH_HAS, body) + payload = data.get("data") if ok else None + if not isinstance(payload, dict) or not isinstance(payload.get("has"), bool): + if raise_on_error: + raise AppApiException(500, f"Invalid Milvus response ({_PATH_HAS}): expected boolean data.has") + return False + if payload["has"]: + MilvusVector._existing.add(knowledge_id) + return True + return False + + def _ensure_collection(self, knowledge_id: str, dim: int): + knowledge_id = str(knowledge_id) + if not dim or dim <= 0: + raise AppApiException(500, _("Invalid Milvus vector dimension")) + if knowledge_id in MilvusVector._loaded: + return + exists = self._has_collection(knowledge_id, raise_on_error=True) + name = self._collection_name(knowledge_id) + body = { + "collectionName": name, + "schema": { + "autoID": False, + "enableDynamicField": False, + "fields": [ + { + "fieldName": "id", + "dataType": "VarChar", + "isPrimary": True, + "elementTypeParams": {"max_length": 128}, + }, + {"fieldName": "embedding", "dataType": "FloatVector", "elementTypeParams": {"dim": dim}}, + {"fieldName": "document_id", "dataType": "VarChar", "elementTypeParams": {"max_length": 128}}, + {"fieldName": "paragraph_id", "dataType": "VarChar", "elementTypeParams": {"max_length": 128}}, + {"fieldName": "is_active", "dataType": "Bool"}, + {"fieldName": "source_id", "dataType": "VarChar", "elementTypeParams": {"max_length": 128}}, + {"fieldName": "source_type", "dataType": "Int8"}, + ], + }, + "indexParams": [ + { + "fieldName": "embedding", + "indexType": "HNSW", + "metricType": "COSINE", + "params": {"M": 16, "efConstruction": 256}, + } + ], + } + if not exists: + self._post_required(_PATH_CREATE, body) + # 也加载已存在的 collection，允许上次创建成功但加载失败后重新尝试。 + self._post_required(_PATH_LOAD, {"collectionName": name}) + MilvusVector._existing.add(knowledge_id) + MilvusVector._loaded.add(knowledge_id) + maxkb_logger.info(f"Prepared Milvus collection for knowledge ID: {knowledge_id} (dim={dim})") + + def vector_is_create(self) -> bool: + # collection 按需懒创建，无需全局初始化 + return True + + def vector_create(self): + return True + + def ensure_knowledge_index(self, knowledge_id: str, dim: int = None): + # 文档向量化时 collection 已在 _batch_save 懒创建；此处仅按给定 dim 幂等创建 + if dim is not None: + self._ensure_collection(knowledge_id, dim) + + def drop_knowledge_index(self, knowledge_id: str): + knowledge_id = str(knowledge_id) + if not self._has_collection(knowledge_id): + return + ok, data = self._post(_PATH_DROP, {"collectionName": self._collection_name(knowledge_id)}) + if ok: + MilvusVector._existing.discard(knowledge_id) + MilvusVector._loaded.discard(knowledge_id) + maxkb_logger.info(f"Dropped Milvus collection for knowledge ID: {knowledge_id}") + else: + maxkb_logger.error(f"Drop Milvus collection failed (kid={knowledge_id}): {data}") + + # ------------------------------------------------------------------ # + # 写入 + # ------------------------------------------------------------------ # + def _save( + self, + text, + source_type, + knowledge_id: str, + document_id: str, + paragraph_id: str, + source_id: str, + is_active: bool, + embedding: MaxKBBaseEmbeddingModel, + ): + self._batch_save( + [ + { + "text": text, + "source_type": source_type, + "knowledge_id": knowledge_id, + "document_id": document_id, + "paragraph_id": paragraph_id, + "source_id": source_id, + "is_active": is_active, + } + ], + embedding, + lambda: False, + ) + + def _batch_save(self, text_list: List[Dict], embedding: MaxKBBaseEmbeddingModel, is_the_task_interrupted): + if not text_list: + return True + texts = [normalize_for_embedding(row.get("text")) for row in text_list] + vectors = embedding.embed_documents(texts) + kid = text_list[0].get("knowledge_id") + dim = len(vectors[0]) if vectors else 0 + self._ensure_collection(kid, dim) + terms = list(QuerySet(Termbase).filter(knowledge_id=kid).values_list("content", flat=True)) + embedding_objs = [] + records = [] + for index, row in enumerate(text_list): + eid = str(uuid.uuid7()) + # PG 元数据 + tsvector（不写 embedding 向量列，向量交由 Milvus） + embedding_objs.append( + Embedding( + id=eid, + document_id=row.get("document_id"), + paragraph_id=row.get("paragraph_id"), + knowledge_id=row.get("knowledge_id"), + is_active=row.get("is_active", True), + source_id=row.get("source_id"), + source_type=row.get("source_type"), + search_vector=SearchVector(Value(to_ts_vector(texts[index], user_words=terms)), config="simple"), + ) + ) + records.append( + { + "id": eid, + "embedding": [float(x) for x in vectors[index]], + "document_id": str(row.get("document_id") or ""), + "paragraph_id": str(row.get("paragraph_id") or ""), + "is_active": bool(row.get("is_active", True)), + "source_id": str(row.get("source_id") or ""), + "source_type": int(row.get("source_type")), + } + ) + if not is_the_task_interrupted(): + QuerySet(Embedding).bulk_create(embedding_objs) if embedding_objs else None + self._upsert_records(kid, records) + return True + + def _upsert_records(self, knowledge_id: str, records: List[Dict]): + if not records: + return + dim = len(records[0]["embedding"]) + self._ensure_collection(knowledge_id, dim) + name = self._collection_name(knowledge_id) + for i in range(0, len(records), 500): + self._post_required(_PATH_UPSERT, {"collectionName": name, "data": records[i : i + 500]}) + + # ------------------------------------------------------------------ # + # 删除 + # ------------------------------------------------------------------ # + def _delete_ids(self, knowledge_id: str, ids: List[str]): + if not ids or not self._has_collection(knowledge_id): + return + name = self._collection_name(knowledge_id) + for i in range(0, len(ids), 1000): + batch = ids[i : i + 1000] + ok, data = self._post(_PATH_DELETE, {"collectionName": name, "filter": f"id in {self._str_list(batch)}"}) + if not ok: + maxkb_logger.error(f"Milvus delete failed (kid={knowledge_id}): {data}") + + def _delete_by_filter(self, filter_kwargs: Dict): + matched = list(QuerySet(Embedding).filter(**filter_kwargs).values("id", "knowledge_id")) + if matched: + by_kid = defaultdict(list) + for r in matched: + by_kid[str(r["knowledge_id"])].append(str(r["id"])) + for kid, ids in by_kid.items(): + self._delete_ids(kid, ids) + QuerySet(Embedding).filter(**filter_kwargs).delete() + + def delete_by_knowledge_id(self, knowledge_id: str): + # 整库向量集中在单一 collection，直接 drop + self.drop_knowledge_index(knowledge_id) + QuerySet(Embedding).filter(knowledge_id=knowledge_id).delete() + + def delete_by_knowledge_id_list(self, knowledge_id_list: List[str]): + for kid in knowledge_id_list: + self.drop_knowledge_index(kid) + QuerySet(Embedding).filter(knowledge_id__in=knowledge_id_list).delete() + + def delete_by_document_id(self, document_id: str): + self._delete_by_filter({"document_id": document_id}) + + def delete_by_document_id_list(self, document_id_list: List[str]): + if not document_id_list: + return True + self._delete_by_filter({"document_id__in": document_id_list}) + return True + + def delete_by_source_id(self, source_id: str, source_type: str): + self._delete_by_filter({"source_id": source_id, "source_type": source_type}) + + def delete_by_source_ids(self, source_ids: List[str], source_type: str): + self._delete_by_filter({"source_id__in": source_ids, "source_type": source_type}) + + def delete_by_paragraph_id(self, paragraph_id: str): + self._delete_by_filter({"paragraph_id": paragraph_id}) + + def delete_by_paragraph_ids(self, paragraph_ids: List[str]): + self._delete_by_filter({"paragraph_id__in": paragraph_ids}) + + # ------------------------------------------------------------------ # + # 更新（PG 元数据 + Milvus 镜像同步） + # ------------------------------------------------------------------ # + def _query_records(self, knowledge_id: str, ids: List[str], output_fields: List[str]) -> List[Dict]: + """按 id 从 Milvus 取记录（含向量），用于更新时回取向量。""" + if not ids or not self._has_collection(knowledge_id): + return [] + ok, data = self._post( + _PATH_QUERY, + { + "collectionName": self._collection_name(knowledge_id), + "filter": f"id in {self._str_list(ids)}", + "outputFields": output_fields, + "limit": len(ids), + }, + ) + if not ok: + return [] + return data.get("data") or [] + + def _update_by_filter(self, filter_kwargs: Dict, instance: Dict): + matched = list( + QuerySet(Embedding) + .filter(**filter_kwargs) + .values("id", "knowledge_id", "document_id", "paragraph_id", "is_active", "source_id", "source_type") + ) + if not matched: + return + # 向量列在 Milvus 模式下不写 PG，从 PG 更新中剔除 + pg_instance = {k: v for k, v in instance.items() if k != "embedding"} + if pg_instance: + QuerySet(Embedding).filter(**filter_kwargs).update(**pg_instance) + self._sync_update(matched, instance) + + def _sync_update(self, matched_rows: List[Dict], instance: Dict): + if not matched_rows: + return + has_embedding = "embedding" in instance + new_embedding = instance.get("embedding") + knowledge_changed = "knowledge_id" in instance + # 未提供新向量时，从 Milvus 取回现有向量用于整体 upsert + emb_map = {} + if not has_embedding: + need = defaultdict(list) + for r in matched_rows: + need[str(r["knowledge_id"])].append(str(r["id"])) + for kid, ids in need.items(): + recs = self._query_records(kid, ids, ["embedding"]) + for rec in recs: + emb_map[str(rec.get("id"))] = rec.get("embedding") + # 按目标 collection 分组 upsert + by_target = defaultdict(list) + for r in matched_rows: + target_kid = str(instance["knowledge_id"]) if knowledge_changed else str(r["knowledge_id"]) + emb = new_embedding if has_embedding else emb_map.get(str(r["id"])) + if not emb: + # Milvus 中尚无该记录（未向量化），跳过镜像 + continue + by_target[target_kid].append( + { + "id": str(r["id"]), + "embedding": [float(x) for x in emb], + "document_id": str(instance.get("document_id", r["document_id"]) or ""), + "paragraph_id": str(r["paragraph_id"]), + "is_active": bool(instance.get("is_active", r["is_active"])), + "source_id": str(instance.get("source_id", r["source_id"]) or ""), + "source_type": int(instance.get("source_type", r["source_type"])), + } + ) + for target_kid, recs in by_target.items(): + self._upsert_records(target_kid, recs) + # 跨知识库迁移时，从旧 collection 删除 + if knowledge_changed: + new_kid = str(instance["knowledge_id"]) + by_source = defaultdict(list) + for r in matched_rows: + if str(r["knowledge_id"]) != new_kid: + by_source[str(r["knowledge_id"])].append(str(r["id"])) + for source_kid, ids in by_source.items(): + self._delete_ids(source_kid, ids) + + def update_by_paragraph_id(self, paragraph_id: str, instance: Dict): + self._update_by_filter({"paragraph_id": paragraph_id}, instance) + + def update_by_paragraph_ids(self, paragraph_ids, instance: Dict): + self._update_by_filter({"paragraph_id__in": paragraph_ids}, instance) + + def update_by_source_id(self, source_id: str, instance: Dict): + self._update_by_filter({"source_id": source_id}, instance) + + def update_by_source_ids(self, source_ids: List[str], instance: Dict): + self._update_by_filter({"source_id__in": source_ids}, instance) + + # ------------------------------------------------------------------ # + # 检索 + # ------------------------------------------------------------------ # + @staticmethod + def _str_list(items) -> str: + safe = [str(i).replace('"', "") for i in items] + return '["' + '","'.join(safe) + '"]' + + def _build_search_expr(self, is_active, document_id_list, exclude_document_id_list, exclude_paragraph_list) -> str: + parts = [f"is_active == {'true' if is_active else 'false'}"] + if document_id_list: + parts.append(f"document_id in {self._str_list(document_id_list)}") + if exclude_document_id_list: + parts.append(f"document_id not in {self._str_list(exclude_document_id_list)}") + if exclude_paragraph_list: + parts.append(f"paragraph_id not in {self._str_list(exclude_paragraph_list)}") + return " and ".join(parts) + + def _milvus_search( + self, + knowledge_id: str, + query_embedding: List[float], + top_n: int, + is_active, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + ) -> List[Dict]: + if not self._has_collection(knowledge_id): + return [] + body = { + "collectionName": self._collection_name(knowledge_id), + "data": [query_embedding], + "annsField": "embedding", + "limit": min(top_n * 10, 500), + "filter": self._build_search_expr( + is_active, document_id_list, exclude_document_id_list, exclude_paragraph_list + ), + "outputFields": ["paragraph_id"], + "searchParams": {"metricType": "COSINE", "params": {"ef": max(64, top_n * 10)}}, + } + ok, data = self._post(_PATH_SEARCH, body) + if not ok: + return [] + hits = data.get("data") or [] + metadata_by_id = { + str(row["id"]): row + for row in QuerySet(Embedding) + .filter(id__in=[str(hit.get("id") or "") for hit in hits]) + .values("id", "paragraph_id", "source_id", "source_type", "meta") + } + out = [] + for h in hits: + metadata = metadata_by_id.get(str(h.get("id") or "")) + if metadata is None: + continue + # COSINE 度量下 distance 即余弦相似度，与 PG 的 (1 - cosine_distance) 口径一致 + out.append( + { + **metadata, + "id": str(h.get("id") or ""), + "paragraph_id": str(metadata["paragraph_id"]), + "similarity": float(h.get("distance", 0)), + } + ) + return out + + def _dispatch_search( + self, + query_text, + query_embedding, + knowledge_id_list, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + is_active, + top_n, + similarity, + search_mode: SearchMode, + ): + if not knowledge_id_list: + return [] + if search_mode == SearchMode.keywords: + return self._search_keywords( + query_text, + query_embedding, + knowledge_id_list, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + is_active, + top_n, + similarity, + ) + if search_mode == SearchMode.blend: + return self._search_blend( + query_text, + query_embedding, + knowledge_id_list, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + is_active, + top_n, + similarity, + ) + # 默认 / embedding + return self._search_embedding( + query_embedding, + knowledge_id_list, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + is_active, + top_n, + similarity, + ) + + def _search_embedding( + self, + query_embedding, + knowledge_id_list, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + is_active, + top_n, + similarity, + ) -> List[Dict]: + all_results = [] + for kid in knowledge_id_list: + all_results.extend( + self._milvus_search( + kid, + query_embedding, + top_n, + is_active, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + ) + ) + # PG 存量兜底召回（向量仍在 PG 的老段，Milvus 无对应 collection 时由此覆盖） + all_results.extend( + pg_vector_recall( + query_embedding, + knowledge_id_list, + is_active, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + top_n, + ) + ) + return self._dedupe_and_rank(all_results, similarity, top_n, use_similarity=True) + + def _search_keywords( + self, + query_text, + query_embedding, + knowledge_id_list, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + is_active, + top_n, + similarity, + ) -> List[Dict]: + # 关键词检索完全复用 PG tsvector（元数据仍在 PG） + qs = QuerySet(Embedding).filter(knowledge_id__in=knowledge_id_list, is_active=is_active) + if document_id_list: + qs = qs.filter(document_id__in=document_id_list) + if exclude_document_id_list: + qs = qs.exclude(document_id__in=exclude_document_id_list) + if exclude_paragraph_list: + qs = qs.exclude(paragraph_id__in=exclude_paragraph_list) + return KeywordsSearch().handle( + qs, query_text, query_embedding, top_n, similarity, SearchMode.keywords, knowledge_id_list + ) + + def _search_blend( + self, + query_text, + query_embedding, + knowledge_id_list, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + is_active, + top_n, + similarity, + ) -> List[Dict]: + # 1. Milvus 向量召回候选 + candidates = [] + for kid in knowledge_id_list: + candidates.extend( + self._milvus_search( + kid, + query_embedding, + top_n, + is_active, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + ) + ) + # PG 存量兜底召回（向量仍在 PG 的老段） + candidates.extend( + pg_vector_recall( + query_embedding, + knowledge_id_list, + is_active, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + top_n, + ) + ) + if not candidates: + return [] + # 2. PG 文本分（ts_rank_cd）回查 + rank_map = text_rank_by_ids([c["id"] for c in candidates], query_text, knowledge_id_list) + # 3. 融合：(1 - distance) + ts_rank_cd，等价于原 blend_search.sql 语义 + for c in candidates: + text_rank = (rank_map.get(c["id"]) or {}).get("text_rank") or 0 + c["comprehensive_score"] = c["similarity"] + float(text_rank) + # 与 PG blend_search.sql 口径对齐：similarity 即 comprehensive_score + c["similarity"] = c["comprehensive_score"] + return self._dedupe_and_rank(candidates, similarity, top_n, use_similarity=False) + + @staticmethod + def _dedupe_and_rank(results: List[Dict], similarity: float, top_n: int, use_similarity: bool) -> List[Dict]: + by_para = {} + for r in results: + pid = r.get("paragraph_id") + if not pid: + continue + score = r.get("similarity") if use_similarity else r.get("comprehensive_score") + cur = by_para.get(pid) + if cur is None or score > (cur.get("similarity") if use_similarity else cur.get("comprehensive_score")): + r["comprehensive_score"] = r.get("comprehensive_score", score) + r["similarity"] = r.get("similarity", score) + by_para[pid] = r + ranked = list(by_para.values()) + ranked = [ + r for r in ranked if (r.get("similarity") if use_similarity else r.get("comprehensive_score")) > similarity + ] + ranked.sort(key=lambda x: x.get("similarity") if use_similarity else x.get("comprehensive_score"), reverse=True) + return ranked[:top_n] + + def query( + self, + query_text: str, + query_embedding: List[float], + knowledge_id_list: list[str], + document_id_list: list[str] | None, + exclude_document_id_list: list[str], + exclude_paragraph_list: list[str], + is_active: bool, + top_n: int, + similarity: float, + search_mode: SearchMode, + ): + return self._dispatch_search( + query_text, + query_embedding, + knowledge_id_list, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + is_active, + top_n, + similarity, + search_mode, + ) + + def hit_test( + self, + query_text, + knowledge_id: list[str], + exclude_document_id_list: list[str], + top_number: int, + similarity: float, + search_mode: SearchMode, + embedding: MaxKBBaseEmbeddingModel, + image_list: list[str] | None = None, + document_id_list: list[str] | None = None, + exclude_paragraph_id_list: list[str] | None = None, + ): + if not knowledge_id or document_id_list == []: + return [] + image_list = list(image_list or []) + query_text = normalize_for_embedding(query_text) + + if image_list and not embedding.supports_image_embedding(): + raise AppApiException(500, _("The current embedding model does not support image embedding")) + + query_units = [] + if search_mode == SearchMode.keywords: + if not query_text: + return [] + query_units.append({"type": "text", "embedding": []}) + else: + if query_text: + query_units.append({"type": "text", "embedding": embedding.embed_query(query_text)}) + if image_list: + image_embeddings = embedding.embed_images(image_list) + if len(image_embeddings) != len(image_list): + raise AppApiException(500, _("The image embedding model returned an incomplete result")) + query_units.extend({"type": "image", "embedding": value} for value in image_embeddings) + + # 与 PG 检索一致：分别查询文本和每张图片，保留每个段落得分最高的命中单元。 + best_result_by_paragraph = {} + for query_unit_index, query_unit in enumerate(query_units): + unit_search_mode = search_mode + if query_unit["type"] == "image" and search_mode == SearchMode.blend: + unit_search_mode = SearchMode.embedding + results = self._dispatch_search( + query_text, + query_unit["embedding"], + knowledge_id, + document_id_list, + exclude_document_id_list, + exclude_paragraph_id_list, + True, + top_number, + similarity, + unit_search_mode, + ) + for result in results: + paragraph_id = str(result.get("paragraph_id")) + score = result.get("comprehensive_score", result.get("similarity", 0)) or 0 + previous = best_result_by_paragraph.get(paragraph_id) + previous_score = ( + previous.get("comprehensive_score", previous.get("similarity", 0)) or 0 + if previous is not None + else 0 + ) + if previous is None or score > previous_score: + best_result_by_paragraph[paragraph_id] = { + **result, + "query_unit_type": query_unit["type"], + "query_unit_index": query_unit_index, + } + + result_list = list(best_result_by_paragraph.values()) + result_list.sort( + key=lambda item: item.get("comprehensive_score", item.get("similarity", 0)) or 0, + reverse=True, + ) + return result_list[:top_number] diff --git a/apps/knowledge/vector/pg_fulltext.py b/apps/knowledge/vector/pg_fulltext.py new file mode 100644 index 00000000000..5032bbcd968 --- /dev/null +++ b/apps/knowledge/vector/pg_fulltext.py @@ -0,0 +1,100 @@ +# coding=utf-8 +""" +@project: maxkb +@file： pg_fulltext.py +@desc: PG 全文检索复用 helper。混合存储下，元数据 + tsvector 仍在 PG， + Milvus 等外部向量库的 keywords / blend 检索可复用这里的 PG 全文能力。 +""" + +import json +import os +from typing import Dict, List + +from common.db.search import generate_sql_by_query_dict +from common.db.sql_execute import select_list +from common.utils.common import get_file_content +from common.utils.ts_vecto_util import to_query +from django.db.models import QuerySet +from knowledge.models import Embedding, Termbase +from maxkb.conf import PROJECT_DIR + + +def text_rank_by_ids(embedding_ids: List[str], query_text: str, knowledge_id_list: list[str] | None) -> Dict[str, dict]: + """ + 给定一批 embedding id，计算其在 PG tsvector 上的文本相关性得分， + 供 blend 检索与 Milvus 向量召回结果融合使用。 + + :param embedding_ids: embedding 主键 id 列表 + :param query_text: 原始查询文本（未分词） + :param knowledge_id_list: 知识库 id 列表，用于术语表加权 + :return: {id: {"id", "paragraph_id", "text_rank"}} + """ + if not embedding_ids: + return {} + query_set = QuerySet(Embedding).filter(id__in=embedding_ids) + exec_sql, exec_params = generate_sql_by_query_dict( + {"embedding_query": query_set}, + select_string=get_file_content(os.path.join(PROJECT_DIR, "apps", "knowledge", "sql", "blend_text_rank.sql")), + with_table_name=True, + ) + terms = ( + list(QuerySet(Termbase).filter(knowledge_id__in=knowledge_id_list).values_list("content", flat=True)) + if knowledge_id_list + else None + ) + rows = select_list(exec_sql, [to_query(query_text, user_words=terms), *exec_params]) + return {row["id"]: row for row in rows} + + +def pg_vector_recall( + query_embedding: List[float], + knowledge_id_list: list[str], + is_active, + document_id_list, + exclude_document_id_list, + exclude_paragraph_list, + top_n: int, +) -> List[Dict]: + """ + PG 向量召回（双查兜底）：按 query_embedding 在 PG embedding 列做最近邻检索， + 返回 PG 向量命中及 source_id、source_type、meta，供图片展示与召回统计使用。 + + Milvus 模式下新增文本向量的 PG embedding 列为 NULL；历史文本及图片向量仍在 PG， + 由此统一召回，再与 Milvus 命中按段落合并。 + """ + if not knowledge_id_list: + return [] + qs = QuerySet(Embedding).filter(knowledge_id__in=knowledge_id_list, is_active=is_active) + if document_id_list: + qs = qs.filter(document_id__in=document_id_list) + if exclude_document_id_list: + qs = qs.exclude(document_id__in=exclude_document_id_list) + if exclude_paragraph_list: + qs = qs.exclude(paragraph_id__in=exclude_paragraph_list) + exec_sql, exec_params = generate_sql_by_query_dict( + {"embedding_query": qs}, + select_string=get_file_content(os.path.join(PROJECT_DIR, "apps", "knowledge", "sql", "embedding_recall.sql")), + with_table_name=True, + ) + rows = select_list( + exec_sql, + [ + len(query_embedding), + json.dumps(query_embedding), + *exec_params, + len(query_embedding), + json.dumps(query_embedding), + top_n, + ], + ) + return [ + { + "id": str(row["id"]), + "paragraph_id": str(row["paragraph_id"]), + "source_id": row["source_id"], + "source_type": row["source_type"], + "meta": row["meta"], + "similarity": float(row["similarity"]), + } + for row in rows + ] diff --git a/apps/knowledge/vector/pg_vector.py b/apps/knowledge/vector/pg_vector.py index 193b9b5fac4..6858c38374f 100644 --- a/apps/knowledge/vector/pg_vector.py +++ b/apps/knowledge/vector/pg_vector.py @@ -19,9 +19,10 @@ from models_provider.base_model_provider import MaxKBBaseEmbeddingModel from common.db.search import generate_sql_by_query_dict -from common.db.sql_execute import select_list +from common.db.sql_execute import select_list, sql_execute, update_execute from common.exception.app_exception import AppApiException from common.utils.common import get_file_content +from common.utils.logger import maxkb_logger from common.utils.ts_vecto_util import to_ts_vector, to_query from knowledge.models import Embedding, SearchMode, SourceType, Termbase from knowledge.vector.base_vector import BaseVectorStore, normalize_for_embedding @@ -57,6 +58,30 @@ def vector_is_create(self) -> bool: def vector_create(self): return True + def ensure_knowledge_index(self, knowledge_id: str, dim: int = None): + # pgvector 按 knowledge_id 建局部 HNSW 索引，超过 2000 维不建 + sql = f"SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'embedding' AND indexname = 'embedding_hnsw_idx_{knowledge_id}'" + index = sql_execute(sql, []) + if not index: + sql = f"SELECT vector_dims(embedding) AS dims FROM embedding WHERE knowledge_id = '{knowledge_id}' LIMIT 1" + result = sql_execute(sql, []) + if len(result) == 0: + return + dims = result[0]["dims"] + # 超过2000维度不创建索引，pgvector hnsw索引不支持超过2000维度 + if dims < 2000: + sql = f"""CREATE INDEX "embedding_hnsw_idx_{knowledge_id}" ON embedding USING hnsw ((embedding::vector({dims})) vector_cosine_ops) WHERE knowledge_id = '{knowledge_id}'""" + update_execute(sql, []) + maxkb_logger.info(f"Created index for knowledge ID: {knowledge_id}") + + def drop_knowledge_index(self, knowledge_id: str): + sql = f"SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'embedding' AND indexname = 'embedding_hnsw_idx_{knowledge_id}'" + index = sql_execute(sql, []) + if index: + sql = f'DROP INDEX "embedding_hnsw_idx_{knowledge_id}"' + update_execute(sql, []) + maxkb_logger.info(f"Dropped index for knowledge ID: {knowledge_id}") + def _save( self, text, diff --git a/apps/maxkb/conf.py b/apps/maxkb/conf.py index 69c524c8519..0db2f9ffff7 100644 --- a/apps/maxkb/conf.py +++ b/apps/maxkb/conf.py @@ -53,6 +53,13 @@ class Config(dict): "S3_SECRET_KEY": "seaweedfsadmin", "S3_ENDPOINT": "http://127.0.0.1:8333", "S3_BUCKET_NAME": "maxkb", + # 向量库：VECTOR_STORE_NAME 显式指定优先；未指定时按 MILVUS_URI 有无自动判断 + # （配了 MILVUS_URI 用 milvus，否则 pg_vector，保证老用户零改动） + "VECTOR_STORE_NAME": "", + # Milvus 连接 URI（host:port 统一在 URI 中，例如 http://127.0.0.1:19530）；配置后自动启用 milvus + "MILVUS_URI": "", + # Milvus 鉴权 token（可选，留空表示无鉴权）；可为 API key 或 user:password 形式 + "MILVUS_TOKEN": "", } def get_debug(self) -> bool: From 13940dc481a2518c809c005b894514bd689c96fe Mon Sep 17 00:00:00 2001 From: CaptainB Date: Fri, 23 Oct 2026 17:45:49 +0800 Subject: [PATCH 2/2] chore: remove obsolete migration files for v2 upgrade --- ...lication_default_model_setting_and_more.py | 22 ++ .../migrations/0015_chat_execute_type.py | 23 -- ...d_messages_chatrecord_question_and_more.py | 30 --- ...setting_and_more.py => 0016_v3_upgrade.py} | 58 ++++- .../0017_application_is_portal_and_more.py | 17 -- .../0019_applicationversion_publish_desc.py | 17 -- apps/common/db/README.md | 50 ++++ apps/common/tests/test_v2_migrations.py | 154 +++++++++++++ .../migrations/0010_file_storage_type.py | 20 -- .../migrations/0010_publicfileaccess.py | 44 ++++ ...workflow_default_model_setting_and_more.py | 22 ++ .../migrations/0011_publicfileaccess.py | 28 --- ...l_knowledge_sync.py => 0012_v3_upgrade.py} | 117 +++++++++- .../migrations/0013_paragraph_asset_recall.py | 22 -- ...workflow_default_model_setting_and_more.py | 44 ---- .../0015_knowledge_external_service.py | 37 --- .../migrations/0016_embedding_nullable.py | 21 -- ...alter_chatuser_nick_name_chatuserapikey.py | 33 --- .../migrations/0006_v3_upgrade.py | 213 ++++++++++++++++++ ...07_workspaceusergroupresourcepermission.py | 35 --- .../0008_add_chat_user_token_quota.py | 86 ------- .../migrations/0002_alter_user_nick_name.py | 18 -- apps/users/migrations/0002_v3_upgrade.py | 95 ++++++++ ...systemusergroup_systemusergrouprelation.py | 41 ---- ...004_alter_systemusergrouprelation_group.py | 19 -- 25 files changed, 765 insertions(+), 501 deletions(-) create mode 100644 apps/application/migrations/0015_application_default_model_setting_and_more.py delete mode 100644 apps/application/migrations/0015_chat_execute_type.py delete mode 100644 apps/application/migrations/0016_chatrecord_messages_chatrecord_question_and_more.py rename apps/application/migrations/{0018_application_default_model_setting_and_more.py => 0016_v3_upgrade.py} (54%) delete mode 100644 apps/application/migrations/0017_application_is_portal_and_more.py delete mode 100644 apps/application/migrations/0019_applicationversion_publish_desc.py create mode 100644 apps/common/db/README.md create mode 100644 apps/common/tests/test_v2_migrations.py delete mode 100644 apps/knowledge/migrations/0010_file_storage_type.py create mode 100644 apps/knowledge/migrations/0010_publicfileaccess.py create mode 100644 apps/knowledge/migrations/0011_knowledgeworkflow_default_model_setting_and_more.py delete mode 100644 apps/knowledge/migrations/0011_publicfileaccess.py rename apps/knowledge/migrations/{0012_multimodal_knowledge_sync.py => 0012_v3_upgrade.py} (77%) delete mode 100644 apps/knowledge/migrations/0013_paragraph_asset_recall.py delete mode 100644 apps/knowledge/migrations/0014_knowledgeworkflow_default_model_setting_and_more.py delete mode 100644 apps/knowledge/migrations/0015_knowledge_external_service.py delete mode 100644 apps/knowledge/migrations/0016_embedding_nullable.py delete mode 100644 apps/system_manage/migrations/0006_alter_chatuser_nick_name_chatuserapikey.py create mode 100644 apps/system_manage/migrations/0006_v3_upgrade.py delete mode 100644 apps/system_manage/migrations/0007_workspaceusergroupresourcepermission.py delete mode 100644 apps/system_manage/migrations/0008_add_chat_user_token_quota.py delete mode 100644 apps/users/migrations/0002_alter_user_nick_name.py create mode 100644 apps/users/migrations/0002_v3_upgrade.py delete mode 100644 apps/users/migrations/0003_systemusergroup_systemusergrouprelation.py delete mode 100644 apps/users/migrations/0004_alter_systemusergrouprelation_group.py diff --git a/apps/application/migrations/0015_application_default_model_setting_and_more.py b/apps/application/migrations/0015_application_default_model_setting_and_more.py new file mode 100644 index 00000000000..d2dada832c5 --- /dev/null +++ b/apps/application/migrations/0015_application_default_model_setting_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.16 on 2026-08-25 05:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("application", "0014_applicationversion_knowledge_ids"), + ] + + operations = [ + migrations.AddField( + model_name="application", + name="default_model_setting", + field=models.JSONField(default=dict, verbose_name="默认模型设置"), + ), + migrations.AddField( + model_name="applicationversion", + name="default_model_setting", + field=models.JSONField(default=dict, verbose_name="默认模型设置"), + ), + ] diff --git a/apps/application/migrations/0015_chat_execute_type.py b/apps/application/migrations/0015_chat_execute_type.py deleted file mode 100644 index 9f0e446d4f9..00000000000 --- a/apps/application/migrations/0015_chat_execute_type.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 5.2.14 on 2026-07-21 08:57 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('application', '0014_applicationversion_knowledge_ids'), - ] - - operations = [ - migrations.AddField( - model_name='chat', - name='execute_type', - field=models.CharField(choices=[('ANONYMOUS_USER', '匿名用户'), ('CHAT_USER', '对话用户'), ('SYSTEM_API_KEY', '系统API_KEY'), ('APPLICATION_API_KEY', '应用API_KEY'), ('PLATFORM_USER', '平台用户')], default='CHAT', max_length=64, verbose_name='执行类型'), - ), - migrations.AddField( - model_name="chatrecord", - name="workflow_context", - field=models.JSONField(blank=True, default=dict, null=True, verbose_name="工作流上下文"), - ), - ] diff --git a/apps/application/migrations/0016_chatrecord_messages_chatrecord_question_and_more.py b/apps/application/migrations/0016_chatrecord_messages_chatrecord_question_and_more.py deleted file mode 100644 index bf835deb315..00000000000 --- a/apps/application/migrations/0016_chatrecord_messages_chatrecord_question_and_more.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 5.2.14 on 2026-07-22 08:33 - -import common.encoder.encoder -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('application', '0015_chat_execute_type'), - ] - - operations = [ - migrations.AddField( - model_name='chatrecord', - name='messages', - field=django.contrib.postgres.fields.ArrayField(base_field=models.JSONField(), default=list, size=None, verbose_name='响应message'), - ), - migrations.AddField( - model_name='chatrecord', - name='question', - field=models.JSONField(default=dict, encoder=common.encoder.encoder.SystemEncoder, verbose_name='用户的消息'), - ), - migrations.AddField( - model_name='chatrecord', - name='version', - field=models.IntegerField(default=1, verbose_name='版本号'), - ), - ] diff --git a/apps/application/migrations/0018_application_default_model_setting_and_more.py b/apps/application/migrations/0016_v3_upgrade.py similarity index 54% rename from apps/application/migrations/0018_application_default_model_setting_and_more.py rename to apps/application/migrations/0016_v3_upgrade.py index ff25649f8ae..b996fc876c9 100644 --- a/apps/application/migrations/0018_application_default_model_setting_and_more.py +++ b/apps/application/migrations/0016_v3_upgrade.py @@ -1,20 +1,67 @@ -# Generated by Django 6.1 on 2026-09-09 09:28 +# Generated by Django 6.1.2 on 2026-10-23 09:12 +import common.encoder.encoder +import django.contrib.postgres.fields from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("application", "0017_application_is_portal_and_more"), + ("application", "0015_application_default_model_setting_and_more"), ] operations = [ + migrations.AddField( + model_name="chat", + name="execute_type", + field=models.CharField( + choices=[ + ("ANONYMOUS_USER", "匿名用户"), + ("CHAT_USER", "对话用户"), + ("SYSTEM_API_KEY", "系统API_KEY"), + ("APPLICATION_API_KEY", "应用API_KEY"), + ("PLATFORM_USER", "平台用户"), + ], + default="CHAT", + max_length=64, + verbose_name="执行类型", + ), + ), + migrations.AddField( + model_name="chatrecord", + name="workflow_context", + field=models.JSONField(blank=True, default=dict, null=True, verbose_name="工作流上下文"), + ), + migrations.AddField( + model_name="chatrecord", + name="messages", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.JSONField(), default=list, verbose_name="响应message" + ), + ), + migrations.AddField( + model_name="chatrecord", + name="question", + field=models.JSONField( + default=dict, encoder=common.encoder.encoder.SystemEncoder, verbose_name="用户的消息" + ), + ), + migrations.AddField( + model_name="chatrecord", + name="version", + field=models.IntegerField(default=1, verbose_name="版本号"), + ), migrations.AddField( + model_name="application", + name="is_portal", + field=models.BooleanField(default=False, verbose_name="是否在门户上架"), + ), + migrations.AlterField( model_name="application", name="default_model_setting", field=models.JSONField(default=dict, verbose_name="默认模型"), ), - migrations.AddField( + migrations.AlterField( model_name="applicationversion", name="default_model_setting", field=models.JSONField(default=dict, verbose_name="默认模型"), @@ -70,4 +117,9 @@ class Migration(migrations.Migration): verbose_name="执行类型", ), ), + migrations.AddField( + model_name="applicationversion", + name="publish_desc", + field=models.CharField(default="", max_length=1024, verbose_name="更新说明"), + ), ] diff --git a/apps/application/migrations/0017_application_is_portal_and_more.py b/apps/application/migrations/0017_application_is_portal_and_more.py deleted file mode 100644 index 8b4dd159ddc..00000000000 --- a/apps/application/migrations/0017_application_is_portal_and_more.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 6.1 on 2026-08-24 08:29 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("application", "0016_chatrecord_messages_chatrecord_question_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="application", - name="is_portal", - field=models.BooleanField(default=False, verbose_name="是否在门户上架"), - ) - ] diff --git a/apps/application/migrations/0019_applicationversion_publish_desc.py b/apps/application/migrations/0019_applicationversion_publish_desc.py deleted file mode 100644 index f9c071ff985..00000000000 --- a/apps/application/migrations/0019_applicationversion_publish_desc.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 6.1 on 2026-09-16 08:54 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("application", "0018_application_default_model_setting_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="applicationversion", - name="publish_desc", - field=models.CharField(default="", max_length=1024, verbose_name="更新说明"), - ), - ] diff --git a/apps/common/db/README.md b/apps/common/db/README.md new file mode 100644 index 00000000000..88bbf9e64c3 --- /dev/null +++ b/apps/common/db/README.md @@ -0,0 +1,50 @@ +# v2 → v3 数据库升级 + +v3 增量迁移按 Django app 合并，统一执行 `python apps/manage.py migrate`。 +原先有多个增量文件的 app 仅保留以下升级入口，18 个原 v3 增量文件已删除； +完整保留 v2 的迁移链： + +| app | v3 入口 | +| --- | --- | +| application | `0016_v3_upgrade` | +| knowledge | `0012_v3_upgrade` | +| users | `0002_v3_upgrade` | +| system_manage | `0006_v3_upgrade` | + +`models_provider/0002_rename_wenxin_provider_to_qianfan`、 +`tools/0009_toolworkflowversion_publish_desc` 和 `portal/0001_initial` +本身各只有一个 v3 迁移，直接保留。 + +合并文件只负责从 v2 升级，按顺序执行普通 Django 迁移操作。 +不识别开发期间旧 v3 增量迁移的执行记录，也不支持续跑部分升级的开发数据库。 +以下 v2 原始迁移按原文件名、原内容保留在各 app 的 `migrations/` 目录： + +- `knowledge.0010_publicfileaccess`：`public_file_access` 表。 +- `knowledge.0011_knowledgeworkflow_default_model_setting_and_more`：知识库工作流及版本的默认模型配置。 +- `application.0015_application_default_model_setting_and_more`：应用及版本的默认模型配置。 + +知识库在 v2 的 `0011` 后执行 `0012_v3_upgrade`； +应用在 v2 的 `0015` 后执行 `0016_v3_upgrade`。 + +已有 v2 数据库由 Django 根据原迁移记录自然跳过已执行的操作；较早 v2 先补齐剩余 v2 +迁移，再执行 v3 增量。无需自定义迁移兼容类、手动改写记录或 `--fake`。 +v3 不再重复创建 `public_file_access` 和 `default_model_setting`；后者仅同步字段显示名称。 +新迁移失败时，当前 app 的变更随事务回滚，可修复原因后重新执行 `migrate`。 + +升级前备份数据库和文件存储，并先在数据库副本上执行迁移、知识库检索和对话验证。 +此迁移处理数据库结构；不将 PG 向量自动搬到 Milvus，也不转换旧工作流的执行逻辑。 + +## 回归验证 + +```bash +python apps/manage.py test common.tests.test_v2_migrations +``` + +迁移回归需要独立的 PostgreSQL 测试实例、创建测试库权限和 `vector` 扩展。 +将 `MAXKB_DB_*` 全部指向该测试实例后运行：旧的 `0002_refresh_collation_reindex` +会额外访问配置中的 `DB_NAME` 和 `root` 库，因此不能使用业务库配置执行迁移测试。 +回归用例在 Django 创建的测试库中回退并重建迁移状态。 +上述三个 v2 原始迁移来自 `origin/v2` 提交 `792a525c5`，应保持原样。 +测试直接用 Django 迁移执行器迁移至 v2 的真实依赖节点，写入历史数据后升级至 v3， +验证 v2 基础版本及较新版本升级、公共文件授权、默认模型配置、对话、向量和 Token 数据保留， +以及重复运行不产生额外迁移。 diff --git a/apps/common/tests/test_v2_migrations.py b/apps/common/tests/test_v2_migrations.py new file mode 100644 index 00000000000..28541a12137 --- /dev/null +++ b/apps/common/tests/test_v2_migrations.py @@ -0,0 +1,154 @@ +"""Upgrade the actual v2 migration graph and preserve representative user data.""" + +from copy import deepcopy + +from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.db.migrations.recorder import MigrationRecorder +from django.test import TransactionTestCase + +V2_BRANCH_MIGRATIONS = [ + ("knowledge", "0010_publicfileaccess"), + ("knowledge", "0011_knowledgeworkflow_default_model_setting_and_more"), + ("application", "0015_application_default_model_setting_and_more"), +] +V2_LATEST = [ + ("knowledge", "0011_knowledgeworkflow_default_model_setting_and_more"), + ("application", "0015_application_default_model_setting_and_more"), + ("users", "0001_initial"), + ("system_manage", "0005_resourcemapping"), + ("models_provider", "0001_initial"), + ("tools", "0008_toolworkflow_default_model_setting_and_more"), + ("portal", None), +] +V3_MIGRATIONS = [ + ("application", "0016_v3_upgrade"), + ("knowledge", "0012_v3_upgrade"), + ("users", "0002_v3_upgrade"), + ("system_manage", "0006_v3_upgrade"), +] + + +class V2MigrationUpgradeTests(TransactionTestCase): + def setUp(self): + executor = MigrationExecutor(connection) + self.latest = executor.loader.graph.leaf_nodes() + self.addCleanup(self.restore_latest) + executor.migrate(V2_LATEST) + self.v2_state = executor.loader.project_state([key for key in V2_LATEST if key[1] is not None]) + + def restore_latest(self): + MigrationExecutor(connection).migrate(self.latest) + + def test_upgrade_preserves_public_access_and_all_default_model_settings(self): + apps = self.v2_state.apps + access = apps.get_model("knowledge", "PublicFileAccess").objects.create( + source_type="KNOWLEDGE", source_id="old-kb" + ) + apps.get_model("application", "ApplicationFolder").objects.get_or_create( + id="default", defaults={"name": "v2 folder", "lft": 1, "rght": 2, "tree_id": 1, "level": 0} + ) + saved = [] + for app_label, model_name in ( + ("application", "Application"), + ("application", "ApplicationVersion"), + ("knowledge", "KnowledgeWorkflow"), + ("knowledge", "KnowledgeWorkflowVersion"), + ): + model = apps.get_model(app_label, model_name) + values = {"default_model_setting": {"model_id": "v2-model", "params": {"temperature": 0.25}}} + # Knowledge references permit orphaned imported resources in v2. + for field in model._meta.local_fields: + if field.is_relation and not field.null: + values[field.attname] = ( + "00000000-0000-0000-0000-000000000001" if field.name != "folder" else "default" + ) + if model_name == "ApplicationVersion": + values["application_id"] = saved[0][2] + obj = model.objects.create(**values) + saved.append((app_label, model_name, obj.pk, deepcopy(values["default_model_setting"]))) + + chat = apps.get_model("application", "Chat").objects.create( + application_id=saved[0][2], chat_user_id="v2-chat-user", abstract="历史对话" + ) + record = apps.get_model("application", "ChatRecord").objects.create( + chat_id=chat.pk, problem_text="旧问题", answer_text="旧答案", index=1, message_tokens=12, answer_tokens=23 + ) + anonymous_chat = apps.get_model("application", "Chat").objects.create(application_id=saved[0][2]) + apps.get_model("application", "ChatRecord").objects.create( + chat_id=anonymous_chat.pk, index=1, message_tokens=3, answer_tokens=4 + ) + embedding = apps.get_model("knowledge", "Embedding").objects.create( + id="v2-vector", + source_id="v2-paragraph", + embedding="[0.1,0.2,0.3]", + document_id="00000000-0000-0000-0000-000000000001", + knowledge_id="00000000-0000-0000-0000-000000000001", + paragraph_id="00000000-0000-0000-0000-000000000001", + meta={"v2": True}, + ) + executor = MigrationExecutor(connection) + executor.migrate(self.latest) + current = executor.loader.project_state(self.latest).apps + self.assertEqual( + current.get_model("knowledge", "PublicFileAccess").objects.get(pk=access.pk).source_id, "old-kb" + ) + for app_label, model_name, pk, value in saved: + self.assertEqual(current.get_model(app_label, model_name).objects.get(pk=pk).default_model_setting, value) + record = current.get_model("application", "ChatRecord").objects.get(pk=record.pk) + self.assertEqual((record.problem_text, record.answer_text), ("旧问题", "旧答案")) + quota = current.get_model("system_manage", "ChatUserTokenQuota").objects.get(user_id="v2-chat-user") + self.assertEqual((quota.used_tokens, quota.total_tokens), (35, 35)) + self.assertFalse( + current.get_model("system_manage", "ChatUserTokenQuota").objects.filter(user_id="None").exists() + ) + embedding = current.get_model("knowledge", "Embedding").objects.get(pk=embedding.pk) + self.assertEqual(embedding.embedding, "[0.1,0.2,0.3]") + self.assertEqual(embedding.meta, {"v2": True}) + with connection.cursor() as cursor: + for table, expected in ( + ("file", "storage_type"), + ("application_chat", "execute_type"), + ("application_chat_record", "messages"), + ): + columns = connection.introspection.get_table_description(cursor, table) + self.assertIn(expected, {column.name for column in columns}) + applied = MigrationRecorder(connection).applied_migrations() + self.assertTrue(set(V2_BRANCH_MIGRATIONS).issubset(applied)) + self.assertTrue(set(V3_MIGRATIONS).issubset(applied)) + executor = MigrationExecutor(connection) + self.assertEqual(executor.migration_plan(self.latest), []) + executor.migrate(self.latest) + + def test_older_v2_database_without_default_model_migrations(self): + targets = [ + ("knowledge", "0010_publicfileaccess"), + ("application", "0014_applicationversion_knowledge_ids"), + *V2_LATEST[2:], + ] + executor = MigrationExecutor(connection) + executor.migrate(targets) + apps = executor.loader.project_state([key for key in targets if key[1] is not None]).apps + row = apps.get_model("knowledge", "PublicFileAccess").objects.create(source_type="FILE", source_id="old-file") + executor = MigrationExecutor(connection) + executor.migrate(self.latest) + current = executor.loader.project_state(self.latest).apps + self.assertEqual( + current.get_model("knowledge", "PublicFileAccess").objects.get(pk=row.pk).source_id, "old-file" + ) + self.assertTrue(set(V2_BRANCH_MIGRATIONS).issubset(MigrationRecorder(connection).applied_migrations())) + self.assertEqual(MigrationExecutor(connection).migration_plan(self.latest), []) + + def test_v2_base_without_branch_specific_migrations(self): + executor = MigrationExecutor(connection) + executor.migrate( + [ + ("knowledge", "0009_document_user"), + ("application", "0014_applicationversion_knowledge_ids"), + *V2_LATEST[2:], + ] + ) + self.assertFalse(set(V2_BRANCH_MIGRATIONS) & MigrationRecorder(connection).applied_migrations().keys()) + MigrationExecutor(connection).migrate(self.latest) + self.assertTrue(set(V2_BRANCH_MIGRATIONS).issubset(MigrationRecorder(connection).applied_migrations())) + self.assertEqual(MigrationExecutor(connection).migration_plan(self.latest), []) diff --git a/apps/knowledge/migrations/0010_file_storage_type.py b/apps/knowledge/migrations/0010_file_storage_type.py deleted file mode 100644 index 905e3450723..00000000000 --- a/apps/knowledge/migrations/0010_file_storage_type.py +++ /dev/null @@ -1,20 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("knowledge", "0009_document_user"), - ] - - operations = [ - migrations.AddField( - model_name="file", - name="storage_type", - field=models.CharField(db_index=True, default="pg", max_length=16), - ), - migrations.AlterField( - model_name="file", - name="loid", - field=models.IntegerField(blank=True, null=True, verbose_name="loid"), - ), - ] diff --git a/apps/knowledge/migrations/0010_publicfileaccess.py b/apps/knowledge/migrations/0010_publicfileaccess.py new file mode 100644 index 00000000000..1d1e34754c9 --- /dev/null +++ b/apps/knowledge/migrations/0010_publicfileaccess.py @@ -0,0 +1,44 @@ +# Generated by Django 5.2.14 on 2026-07-10 10:44 + +import uuid_utils.compat +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("knowledge", "0009_document_user"), + ] + + operations = [ + migrations.CreateModel( + name="PublicFileAccess", + fields=[ + ("create_time", models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="创建时间")), + ("update_time", models.DateTimeField(auto_now=True, db_index=True, verbose_name="修改时间")), + ( + "id", + models.UUIDField( + default=uuid_utils.compat.uuid7, + editable=False, + primary_key=True, + serialize=False, + verbose_name="主键id", + ), + ), + ( + "source_type", + models.CharField( + choices=[("FILE", "文件"), ("APPLICATION", "应用"), ("KNOWLEDGE", "知识库")], + db_index=True, + max_length=20, + verbose_name="资源类型", + ), + ), + ("source_id", models.CharField(db_index=True, max_length=128, verbose_name="资源ID")), + ], + options={ + "db_table": "public_file_access", + "indexes": [models.Index(fields=["source_type", "source_id"], name="public_file_source__a961f4_idx")], + }, + ), + ] diff --git a/apps/knowledge/migrations/0011_knowledgeworkflow_default_model_setting_and_more.py b/apps/knowledge/migrations/0011_knowledgeworkflow_default_model_setting_and_more.py new file mode 100644 index 00000000000..8aec9f78c34 --- /dev/null +++ b/apps/knowledge/migrations/0011_knowledgeworkflow_default_model_setting_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.16 on 2026-08-26 12:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("knowledge", "0010_publicfileaccess"), + ] + + operations = [ + migrations.AddField( + model_name="knowledgeworkflow", + name="default_model_setting", + field=models.JSONField(default=dict, verbose_name="默认模型设置"), + ), + migrations.AddField( + model_name="knowledgeworkflowversion", + name="default_model_setting", + field=models.JSONField(default=dict, verbose_name="默认模型设置"), + ), + ] diff --git a/apps/knowledge/migrations/0011_publicfileaccess.py b/apps/knowledge/migrations/0011_publicfileaccess.py deleted file mode 100644 index fc19f4eb9b3..00000000000 --- a/apps/knowledge/migrations/0011_publicfileaccess.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 5.2.14 on 2026-07-10 10:44 - -import uuid_utils.compat -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('knowledge', '0010_file_storage_type'), - ] - - operations = [ - migrations.CreateModel( - name='PublicFileAccess', - fields=[ - ('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')), - ('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')), - ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')), - ('source_type', models.CharField(choices=[('FILE', '文件'), ('APPLICATION', '应用'), ('KNOWLEDGE', '知识库')], db_index=True, max_length=20, verbose_name='资源类型')), - ('source_id', models.CharField(db_index=True, max_length=128, verbose_name='资源ID')), - ], - options={ - 'db_table': 'public_file_access', - 'indexes': [models.Index(fields=['source_type', 'source_id'], name='public_file_source__a961f4_idx')], - }, - ), - ] diff --git a/apps/knowledge/migrations/0012_multimodal_knowledge_sync.py b/apps/knowledge/migrations/0012_v3_upgrade.py similarity index 77% rename from apps/knowledge/migrations/0012_multimodal_knowledge_sync.py rename to apps/knowledge/migrations/0012_v3_upgrade.py index 9db376e8b6e..3669cb3f61a 100644 --- a/apps/knowledge/migrations/0012_multimodal_knowledge_sync.py +++ b/apps/knowledge/migrations/0012_v3_upgrade.py @@ -1,20 +1,34 @@ -# Generated by Django 6.1 on 2026-08-13 08:39 +# Generated by Django 6.1.2 on 2026-10-23 09:12 import django.db.models.deletion +import knowledge.models.knowledge import uuid_utils.compat from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("knowledge", "0011_publicfileaccess"), + ("knowledge", "0011_knowledgeworkflow_default_model_setting_and_more"), ] operations = [ + migrations.AddField( + model_name="file", + name="storage_type", + field=models.CharField(db_index=True, default="pg", max_length=16), + ), + migrations.AlterField( + model_name="file", + name="loid", + field=models.IntegerField(blank=True, null=True, verbose_name="loid"), + ), migrations.CreateModel( name="KnowledgeSyncLog", fields=[ - ("create_time", models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="创建时间")), + ( + "create_time", + models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="创建时间"), + ), ("update_time", models.DateTimeField(auto_now=True, db_index=True, verbose_name="修改时间")), ( "id", @@ -30,7 +44,11 @@ class Migration(migrations.Migration): ( "sync_type", models.CharField( - choices=[("incremental", "增量同步"), ("replace", "替换同步"), ("complete", "完整同步")], + choices=[ + ("incremental", "增量同步"), + ("replace", "替换同步"), + ("complete", "完整同步"), + ], default="incremental", max_length=16, verbose_name="同步方式", @@ -71,7 +89,10 @@ class Migration(migrations.Migration): migrations.CreateModel( name="ParagraphAsset", fields=[ - ("create_time", models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="创建时间")), + ( + "create_time", + models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="创建时间"), + ), ("update_time", models.DateTimeField(auto_now=True, db_index=True, verbose_name="修改时间")), ( "id", @@ -120,7 +141,11 @@ class Migration(migrations.Migration): ( "sync_state", models.CharField( - choices=[("active", "正常"), ("remote_deleted", "远端已删除"), ("conflict", "同步冲突")], + choices=[ + ("active", "正常"), + ("remote_deleted", "远端已删除"), + ("conflict", "同步冲突"), + ], default="active", max_length=24, verbose_name="同步状态", @@ -141,7 +166,10 @@ class Migration(migrations.Migration): ), ), ("process_error", models.TextField(default="", verbose_name="处理错误")), - ("visual_strategy_hash", models.CharField(default="", max_length=64, verbose_name="图片处理策略哈希")), + ( + "visual_strategy_hash", + models.CharField(default="", max_length=64, verbose_name="图片处理策略哈希"), + ), ("meta", models.JSONField(default=dict, verbose_name="元数据")), ], options={ @@ -383,4 +411,79 @@ class Migration(migrations.Migration): name="uniq_document_asset_source_key", ), ), + migrations.AddField( + model_name="paragraphasset", + name="hit_num", + field=models.IntegerField(db_index=True, default=0, verbose_name="召回次数"), + ), + migrations.AddField( + model_name="paragraphasset", + name="last_hit_time", + field=models.DateTimeField(blank=True, db_index=True, null=True, verbose_name="最后一次召回时间"), + ), + migrations.AlterField( + model_name="knowledgeworkflow", + name="default_model_setting", + field=models.JSONField(default=dict, verbose_name="默认模型"), + ), + migrations.AlterField( + model_name="knowledgeworkflowversion", + name="default_model_setting", + field=models.JSONField(default=dict, verbose_name="默认模型"), + ), + migrations.AlterField( + model_name="file", + name="source_type", + field=models.CharField( + choices=[ + ("KNOWLEDGE", "Knowledge"), + ("APPLICATION", "Application"), + ("TOOL", "Tool"), + ("DOCUMENT", "Document"), + ("PARAGRAPH", "Paragraph"), + ("CHAT", "Chat"), + ("SYSTEM", "System"), + ("TEMPORARY_30_MINUTE", "Temporary 30 Minute"), + ("TEMPORARY_120_MINUTE", "Temporary 120 Minute"), + ("TEMPORARY_1_DAY", "Temporary 1 Day"), + ("APPLICATION_SETTINGS", "Application Settings"), + ], + db_index=True, + default="TEMPORARY_120_MINUTE", + verbose_name="资源类型", + ), + ), + migrations.AddField( + model_name="knowledge", + name="external_service", + field=models.JSONField(default=dict, verbose_name="外部检索服务"), + ), + migrations.AlterField( + model_name="knowledge", + name="external_service", + field=models.JSONField( + default=knowledge.models.knowledge.default_external_service, verbose_name="外部检索服务" + ), + ), + migrations.AlterField( + model_name="knowledgesynclog", + name="status", + field=models.CharField( + choices=[ + ("running", "同步中"), + ("success", "同步成功"), + ("failure", "同步失败"), + ("skipped", "已跳过"), + ], + db_index=True, + default="running", + max_length=16, + verbose_name="同步状态", + ), + ), + migrations.AlterField( + model_name="embedding", + name="embedding", + field=knowledge.models.knowledge.VectorField(null=True, verbose_name="向量"), + ), ] diff --git a/apps/knowledge/migrations/0013_paragraph_asset_recall.py b/apps/knowledge/migrations/0013_paragraph_asset_recall.py deleted file mode 100644 index 5e4444c67e2..00000000000 --- a/apps/knowledge/migrations/0013_paragraph_asset_recall.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 6.1 on 2026-09-03 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("knowledge", "0012_multimodal_knowledge_sync"), - ] - - operations = [ - migrations.AddField( - model_name="paragraphasset", - name="hit_num", - field=models.IntegerField(db_index=True, default=0, verbose_name="召回次数"), - ), - migrations.AddField( - model_name="paragraphasset", - name="last_hit_time", - field=models.DateTimeField(blank=True, db_index=True, null=True, verbose_name="最后一次召回时间"), - ), - ] diff --git a/apps/knowledge/migrations/0014_knowledgeworkflow_default_model_setting_and_more.py b/apps/knowledge/migrations/0014_knowledgeworkflow_default_model_setting_and_more.py deleted file mode 100644 index d50285e332e..00000000000 --- a/apps/knowledge/migrations/0014_knowledgeworkflow_default_model_setting_and_more.py +++ /dev/null @@ -1,44 +0,0 @@ -# Generated by Django 6.1 on 2026-09-11 06:34 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("knowledge", "0013_paragraph_asset_recall"), - ] - - operations = [ - migrations.AddField( - model_name="knowledgeworkflow", - name="default_model_setting", - field=models.JSONField(default=dict, verbose_name="默认模型"), - ), - migrations.AddField( - model_name="knowledgeworkflowversion", - name="default_model_setting", - field=models.JSONField(default=dict, verbose_name="默认模型"), - ), - migrations.AlterField( - model_name="file", - name="source_type", - field=models.CharField( - choices=[ - ("KNOWLEDGE", "Knowledge"), - ("APPLICATION", "Application"), - ("TOOL", "Tool"), - ("DOCUMENT", "Document"), - ("PARAGRAPH", "Paragraph"), - ("CHAT", "Chat"), - ("SYSTEM", "System"), - ("TEMPORARY_30_MINUTE", "Temporary 30 Minute"), - ("TEMPORARY_120_MINUTE", "Temporary 120 Minute"), - ("TEMPORARY_1_DAY", "Temporary 1 Day"), - ("APPLICATION_SETTINGS", "Application Settings"), - ], - db_index=True, - default="TEMPORARY_120_MINUTE", - verbose_name="资源类型", - ), - ), - ] diff --git a/apps/knowledge/migrations/0015_knowledge_external_service.py b/apps/knowledge/migrations/0015_knowledge_external_service.py deleted file mode 100644 index e61e86298b3..00000000000 --- a/apps/knowledge/migrations/0015_knowledge_external_service.py +++ /dev/null @@ -1,37 +0,0 @@ -from django.db import migrations, models - -from knowledge.models.knowledge import default_external_service - - -class Migration(migrations.Migration): - dependencies = [("knowledge", "0014_knowledgeworkflow_default_model_setting_and_more")] - - operations = [ - # An absent authentication setting preserves existing internal access rules. - migrations.AddField( - model_name="knowledge", - name="external_service", - field=models.JSONField(default=dict, verbose_name="外部检索服务"), - ), - migrations.AlterField( - model_name="knowledge", - name="external_service", - field=models.JSONField(default=default_external_service, verbose_name="外部检索服务"), - ), - migrations.AlterField( - model_name="knowledgesynclog", - name="status", - field=models.CharField( - choices=[ - ("running", "同步中"), - ("success", "同步成功"), - ("failure", "同步失败"), - ("skipped", "已跳过"), - ], - db_index=True, - default="running", - max_length=16, - verbose_name="同步状态", - ), - ), - ] diff --git a/apps/knowledge/migrations/0016_embedding_nullable.py b/apps/knowledge/migrations/0016_embedding_nullable.py deleted file mode 100644 index a32b9a35021..00000000000 --- a/apps/knowledge/migrations/0016_embedding_nullable.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated for Milvus vector store support - -from django.db import migrations - -import knowledge.models.knowledge - - -class Migration(migrations.Migration): - dependencies = [ - ("knowledge", "0015_knowledge_external_service"), - ] - - operations = [ - # Milvus 混合存储模式下，向量交由 Milvus 保存，PG 的 embedding 列允许为空。 - # pg_vector 模式不受影响（仍写入真实向量）。 - migrations.AlterField( - model_name="embedding", - name="embedding", - field=knowledge.models.knowledge.VectorField(null=True, verbose_name="向量"), - ), - ] diff --git a/apps/system_manage/migrations/0006_alter_chatuser_nick_name_chatuserapikey.py b/apps/system_manage/migrations/0006_alter_chatuser_nick_name_chatuserapikey.py deleted file mode 100644 index b6631fcba9b..00000000000 --- a/apps/system_manage/migrations/0006_alter_chatuser_nick_name_chatuserapikey.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 6.0.7 on 2026-08-03 02:04 - -import django.db.models.deletion -import uuid_utils.compat -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('system_manage', '0005_resourcemapping'), - ] - - operations = [ - migrations.AlterField( - model_name='chatuser', - name='nick_name', - field=models.CharField(db_index=True, max_length=150, verbose_name='昵称'), - ), - migrations.CreateModel( - name='ChatUserApiKey', - fields=[ - ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')), - ('secret_key', models.CharField(max_length=1024, unique=True, verbose_name='秘钥')), - ('is_active', models.BooleanField(default=True, verbose_name='是否开启')), - ('create_time', models.DateTimeField(auto_now_add=True, db_index=True, null=True, verbose_name='创建时间')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='system_manage.chatuser', verbose_name='用户')), - ], - options={ - 'db_table': 'chat_user_api_key', - }, - ), - ] diff --git a/apps/system_manage/migrations/0006_v3_upgrade.py b/apps/system_manage/migrations/0006_v3_upgrade.py new file mode 100644 index 00000000000..b338cd7796f --- /dev/null +++ b/apps/system_manage/migrations/0006_v3_upgrade.py @@ -0,0 +1,213 @@ +# Generated by Django 6.1.2 on 2026-10-23 09:12 + +import django.contrib.postgres.fields +import django.db.migrations.operations.special +import django.db.models.deletion +import uuid_utils.compat +from django.db import migrations, models + + +def migrate_historical_tokens(apps, schema_editor): + ChatRecord = apps.get_model("application", "ChatRecord") + Chat = apps.get_model("application", "Chat") + ChatUserTokenQuota = apps.get_model("system_manage", "ChatUserTokenQuota") + alias = schema_editor.connection.alias + + chat_user_map = { + str(c["id"]): str(c["chat_user_id"]) + for c in Chat.objects.using(alias).values("id", "chat_user_id").iterator() + if c["chat_user_id"] is not None + } + user_totals = {} + queryset = ChatRecord.objects.using(alias).filter(message_tokens__isnull=False) | ChatRecord.objects.using( + alias + ).filter(answer_tokens__isnull=False) + for record in queryset.values("chat_id", "message_tokens", "answer_tokens").iterator(chunk_size=5000): + user_id = chat_user_map.get(str(record["chat_id"])) + if not user_id: + continue + tokens = (record["message_tokens"] or 0) + (record["answer_tokens"] or 0) + user_totals[user_id] = user_totals.get(user_id, 0) + tokens + + objs = [ + ChatUserTokenQuota(user_id=uid, total_tokens=total, used_tokens=total) for uid, total in user_totals.items() + ] + if objs: + ChatUserTokenQuota.objects.using(alias).bulk_create(objs, batch_size=500) + + +class Migration(migrations.Migration): + dependencies = [ + ("system_manage", "0005_resourcemapping"), + ("users", "0002_v3_upgrade"), + ] + + operations = [ + migrations.AlterField( + model_name="chatuser", + name="nick_name", + field=models.CharField(db_index=True, max_length=150, verbose_name="昵称"), + ), + migrations.CreateModel( + name="ChatUserApiKey", + fields=[ + ( + "id", + models.UUIDField( + default=uuid_utils.compat.uuid7, + editable=False, + primary_key=True, + serialize=False, + verbose_name="主键id", + ), + ), + ("secret_key", models.CharField(max_length=1024, unique=True, verbose_name="秘钥")), + ("is_active", models.BooleanField(default=True, verbose_name="是否开启")), + ( + "create_time", + models.DateTimeField(auto_now_add=True, db_index=True, null=True, verbose_name="创建时间"), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="system_manage.chatuser", + verbose_name="用户", + ), + ), + ], + options={ + "db_table": "chat_user_api_key", + }, + ), + migrations.CreateModel( + name="WorkspaceUserGroupResourcePermission", + fields=[ + ( + "id", + models.UUIDField( + default=uuid_utils.compat.uuid7, + editable=False, + primary_key=True, + serialize=False, + verbose_name="主键id", + ), + ), + ( + "workspace_id", + models.CharField(db_index=True, default="default", max_length=128, verbose_name="工作空间id"), + ), + ( + "auth_target_type", + models.CharField( + choices=[ + ("KNOWLEDGE", "知识库"), + ("APPLICATION", "应用"), + ("TOOL", "工具"), + ("MODEL", "模型"), + ], + db_index=True, + default="KNOWLEDGE", + max_length=128, + verbose_name="授权目标", + ), + ), + ("target", models.CharField(db_index=True, max_length=128, verbose_name="知识库/应用id")), + ( + "auth_type", + models.CharField( + choices=[("ROLE", "Role"), ("RESOURCE_PERMISSION_GROUP", "Resource Permission Group")], + db_default="ROLE", + db_index=True, + default=False, + verbose_name="授权类型", + ), + ), + ( + "permission_list", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + blank=True, + choices=[("VIEW", "View"), ("MANAGE", "Manage"), ("ROLE", "Role")], + default="VIEW", + max_length=256, + ), + default=list, + verbose_name="权限列表", + ), + ), + ( + "create_time", + models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="创建时间"), + ), + ("update_time", models.DateTimeField(auto_now=True, db_index=True, verbose_name="修改时间")), + ( + "user_group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="users.systemusergroup", + verbose_name="用户组id", + ), + ), + ], + options={ + "db_table": "workspace_user_group_resource_permission", + "unique_together": {("workspace_id", "user_group", "auth_target_type", "target")}, + }, + ), + migrations.CreateModel( + name="ChatUserTokenQuota", + fields=[ + ( + "create_time", + models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="创建时间"), + ), + ("update_time", models.DateTimeField(auto_now=True, db_index=True, verbose_name="修改时间")), + ( + "id", + models.UUIDField( + default=uuid_utils.compat.uuid7, + editable=False, + primary_key=True, + serialize=False, + verbose_name="主键id", + ), + ), + ( + "user_id", + models.CharField(db_index=True, max_length=128, unique=True, verbose_name="用户id"), + ), + ( + "quota_type", + models.CharField( + choices=[("UNLIMITED", "不限额"), ("PERIODIC", "按周期限制")], + default="UNLIMITED", + max_length=20, + verbose_name="配额模式", + ), + ), + ( + "period_type", + models.CharField( + blank=True, + choices=[("DAY", "天"), ("WEEK", "周"), ("MONTH", "月")], + max_length=10, + null=True, + verbose_name="周期单位", + ), + ), + ("period_value", models.PositiveIntegerField(blank=True, null=True, verbose_name="周期数量")), + ("token_limit", models.BigIntegerField(blank=True, null=True, verbose_name="Tokens上限")), + ("used_tokens", models.BigIntegerField(default=0, verbose_name="当前周期已使用Tokens")), + ("total_tokens", models.BigIntegerField(default=0, verbose_name="累计Tokens")), + ("period_end", models.DateTimeField(blank=True, null=True, verbose_name="当前周期结束时间")), + ], + options={ + "db_table": "chat_user_token_quota", + }, + ), + migrations.RunPython( + code=migrate_historical_tokens, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + ] diff --git a/apps/system_manage/migrations/0007_workspaceusergroupresourcepermission.py b/apps/system_manage/migrations/0007_workspaceusergroupresourcepermission.py deleted file mode 100644 index e81ae7c6255..00000000000 --- a/apps/system_manage/migrations/0007_workspaceusergroupresourcepermission.py +++ /dev/null @@ -1,35 +0,0 @@ -# Generated by Django 5.2.14 on 2026-08-05 07:06 - -import django.contrib.postgres.fields -import django.db.models.deletion -import uuid_utils.compat -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('system_manage', '0006_alter_chatuser_nick_name_chatuserapikey'), - ('users', '0003_systemusergroup_systemusergrouprelation'), - ] - - operations = [ - migrations.CreateModel( - name='WorkspaceUserGroupResourcePermission', - fields=[ - ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')), - ('workspace_id', models.CharField(db_index=True, default='default', max_length=128, verbose_name='工作空间id')), - ('auth_target_type', models.CharField(choices=[('KNOWLEDGE', '知识库'), ('APPLICATION', '应用'), ('TOOL', '工具'), ('MODEL', '模型')], db_index=True, default='KNOWLEDGE', max_length=128, verbose_name='授权目标')), - ('target', models.CharField(db_index=True, max_length=128, verbose_name='知识库/应用id')), - ('auth_type', models.CharField(choices=[('ROLE', 'Role'), ('RESOURCE_PERMISSION_GROUP', 'Resource Permission Group')], db_default='ROLE', db_index=True, default=False, verbose_name='授权类型')), - ('permission_list', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, choices=[('VIEW', 'View'), ('MANAGE', 'Manage'), ('ROLE', 'Role')], default='VIEW', max_length=256), default=list, size=None, verbose_name='权限列表')), - ('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')), - ('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')), - ('user_group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='users.systemusergroup', verbose_name='用户组id')), - ], - options={ - 'db_table': 'workspace_user_group_resource_permission', - 'unique_together': {('workspace_id', 'user_group', 'auth_target_type', 'target')}, - }, - ), - ] diff --git a/apps/system_manage/migrations/0008_add_chat_user_token_quota.py b/apps/system_manage/migrations/0008_add_chat_user_token_quota.py deleted file mode 100644 index 3a78cbf1ef7..00000000000 --- a/apps/system_manage/migrations/0008_add_chat_user_token_quota.py +++ /dev/null @@ -1,86 +0,0 @@ -# Generated by Django 6.0.7 on 2026-08-06 02:40 - -import uuid_utils.compat -from django.db import migrations, models - - -def migrate_historical_tokens(apps, schema_editor): - ChatRecord = apps.get_model("application", "ChatRecord") - Chat = apps.get_model("application", "Chat") - ChatUserTokenQuota = apps.get_model("system_manage", "ChatUserTokenQuota") - - chat_user_map = {str(c["id"]): str(c["chat_user_id"]) for c in Chat.objects.values("id", "chat_user_id").iterator()} - user_totals = {} - queryset = ChatRecord.objects.filter(message_tokens__isnull=False) | ChatRecord.objects.filter( - answer_tokens__isnull=False - ) - for record in queryset.values("chat_id", "message_tokens", "answer_tokens").iterator(chunk_size=5000): - user_id = chat_user_map.get(str(record["chat_id"])) - if not user_id: - continue - tokens = (record["message_tokens"] or 0) + (record["answer_tokens"] or 0) - user_totals[user_id] = user_totals.get(user_id, 0) + tokens - - objs = [ - ChatUserTokenQuota(user_id=uid, total_tokens=total, used_tokens=total) for uid, total in user_totals.items() - ] - if objs: - ChatUserTokenQuota.objects.bulk_create(objs, batch_size=500) - - -class Migration(migrations.Migration): - dependencies = [ - ("system_manage", "0007_workspaceusergroupresourcepermission"), - ] - - operations = [ - migrations.CreateModel( - name="ChatUserTokenQuota", - fields=[ - ("create_time", models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="创建时间")), - ("update_time", models.DateTimeField(auto_now=True, db_index=True, verbose_name="修改时间")), - ( - "id", - models.UUIDField( - default=uuid_utils.compat.uuid7, - editable=False, - primary_key=True, - serialize=False, - verbose_name="主键id", - ), - ), - ("user_id", models.CharField(db_index=True, max_length=128, unique=True, verbose_name="用户id")), - ( - "quota_type", - models.CharField( - choices=[("UNLIMITED", "不限额"), ("PERIODIC", "按周期限制")], - default="UNLIMITED", - max_length=20, - verbose_name="配额模式", - ), - ), - ( - "period_type", - models.CharField( - blank=True, - choices=[("DAY", "天"), ("WEEK", "周"), ("MONTH", "月")], - max_length=10, - null=True, - verbose_name="周期单位", - ), - ), - ("period_value", models.PositiveIntegerField(blank=True, null=True, verbose_name="周期数量")), - ("token_limit", models.BigIntegerField(blank=True, null=True, verbose_name="Tokens上限")), - ("used_tokens", models.BigIntegerField(default=0, verbose_name="当前周期已使用Tokens")), - ("total_tokens", models.BigIntegerField(default=0, verbose_name="累计Tokens")), - ("period_end", models.DateTimeField(blank=True, null=True, verbose_name="当前周期结束时间")), - ], - options={ - "db_table": "chat_user_token_quota", - }, - ), - migrations.RunPython( - migrate_historical_tokens, - migrations.RunPython.noop, - ), - ] diff --git a/apps/users/migrations/0002_alter_user_nick_name.py b/apps/users/migrations/0002_alter_user_nick_name.py deleted file mode 100644 index 0486dd18de2..00000000000 --- a/apps/users/migrations/0002_alter_user_nick_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 6.0.6 on 2026-07-07 01:46 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('users', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='user', - name='nick_name', - field=models.CharField(db_index=True, max_length=150, verbose_name='昵称'), - ), - ] diff --git a/apps/users/migrations/0002_v3_upgrade.py b/apps/users/migrations/0002_v3_upgrade.py new file mode 100644 index 00000000000..86c9ac23593 --- /dev/null +++ b/apps/users/migrations/0002_v3_upgrade.py @@ -0,0 +1,95 @@ +# Generated by Django 6.1.2 on 2026-10-23 09:12 + +import django.db.models.deletion +import uuid_utils.compat +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("users", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="nick_name", + field=models.CharField(db_index=True, max_length=150, verbose_name="昵称"), + ), + migrations.CreateModel( + name="SystemUserGroup", + fields=[ + ( + "id", + models.CharField( + default=uuid_utils.compat.uuid7, + editable=False, + max_length=128, + primary_key=True, + serialize=False, + verbose_name="主键id", + ), + ), + ("name", models.CharField(db_index=True, max_length=150, unique=True, verbose_name="名称")), + ( + "workspace_id", + models.CharField(db_index=True, default="default", max_length=64, verbose_name="工作空间id"), + ), + ( + "create_time", + models.DateTimeField(auto_now_add=True, db_index=True, null=True, verbose_name="创建时间"), + ), + ( + "update_time", + models.DateTimeField(auto_now=True, db_index=True, null=True, verbose_name="修改时间"), + ), + ], + options={ + "db_table": "system_user_group", + "unique_together": {("workspace_id", "name")}, + }, + ), + migrations.CreateModel( + name="SystemUserGroupRelation", + fields=[ + ( + "id", + models.UUIDField( + default=uuid_utils.compat.uuid7, + editable=False, + primary_key=True, + serialize=False, + verbose_name="主键id", + ), + ), + ( + "group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="users.systemusergroup", + verbose_name="用户组", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="users.user", verbose_name="用户" + ), + ), + ], + options={ + "db_table": "system_user_group_relation", + "constraints": [models.UniqueConstraint(fields=("user", "group"), name="uniq_user_group_relation")], + }, + ), + migrations.AlterField( + model_name="systemusergrouprelation", + name="group", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="user_relations", + to="users.systemusergroup", + verbose_name="用户组", + ), + ), + ] diff --git a/apps/users/migrations/0003_systemusergroup_systemusergrouprelation.py b/apps/users/migrations/0003_systemusergroup_systemusergrouprelation.py deleted file mode 100644 index 951c1192f83..00000000000 --- a/apps/users/migrations/0003_systemusergroup_systemusergrouprelation.py +++ /dev/null @@ -1,41 +0,0 @@ -# Generated by Django 5.2.14 on 2026-08-05 07:06 - -import django.db.models.deletion -import uuid_utils.compat -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('users', '0002_alter_user_nick_name'), - ] - - operations = [ - migrations.CreateModel( - name='SystemUserGroup', - fields=[ - ('id', models.CharField(default=uuid_utils.compat.uuid7, editable=False, max_length=128, primary_key=True, serialize=False, verbose_name='主键id')), - ('name', models.CharField(db_index=True, max_length=150, unique=True, verbose_name='名称')), - ('workspace_id', models.CharField(db_index=True, default='default', max_length=64, verbose_name='工作空间id')), - ('create_time', models.DateTimeField(auto_now_add=True, db_index=True, null=True, verbose_name='创建时间')), - ('update_time', models.DateTimeField(auto_now=True, db_index=True, null=True, verbose_name='修改时间')), - ], - options={ - 'db_table': 'system_user_group', - 'unique_together': {('workspace_id', 'name')}, - }, - ), - migrations.CreateModel( - name='SystemUserGroupRelation', - fields=[ - ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='users.systemusergroup', verbose_name='用户组')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='users.user', verbose_name='用户')), - ], - options={ - 'db_table': 'system_user_group_relation', - 'constraints': [models.UniqueConstraint(fields=('user', 'group'), name='uniq_user_group_relation')], - }, - ), - ] diff --git a/apps/users/migrations/0004_alter_systemusergrouprelation_group.py b/apps/users/migrations/0004_alter_systemusergrouprelation_group.py deleted file mode 100644 index 5bce7cc1b5a..00000000000 --- a/apps/users/migrations/0004_alter_systemusergrouprelation_group.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 6.0.7 on 2026-08-06 02:13 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('users', '0003_systemusergroup_systemusergrouprelation'), - ] - - operations = [ - migrations.AlterField( - model_name='systemusergrouprelation', - name='group', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='user_relations', to='users.systemusergroup', verbose_name='用户组'), - ), - ]