dify/api/tests/integration_tests/vdb/tablestore/test_tablestore.py

import os
import uuid

import tablestore
from _pytest.python_api import approx

from core.rag.datasource.vdb.tablestore.tablestore_vector import (
    TableStoreConfig,
    TableStoreVector,
)
from tests.integration_tests.vdb.test_vector_store import (
    AbstractVectorTest,
    get_example_document,
    get_example_text,
    setup_mock_redis,
)


class TableStoreVectorTest(AbstractVectorTest):
    def __init__(self, normalize_full_text_score: bool = False):
        super().__init__()
        self.vector = TableStoreVector(
            collection_name=self.collection_name,
            config=TableStoreConfig(
                endpoint=os.getenv("TABLESTORE_ENDPOINT"),
                instance_name=os.getenv("TABLESTORE_INSTANCE_NAME"),
                access_key_id=os.getenv("TABLESTORE_ACCESS_KEY_ID"),
                access_key_secret=os.getenv("TABLESTORE_ACCESS_KEY_SECRET"),
                normalize_full_text_bm25_score=normalize_full_text_score,
            ),
        )

    def get_ids_by_metadata_field(self):
        ids = self.vector.get_ids_by_metadata_field(key="doc_id", value=self.example_doc_id)
        assert ids is not None
        assert len(ids) == 1
        assert ids[0] == self.example_doc_id

    def create_vector(self):
        self.vector.create(
            texts=[get_example_document(doc_id=self.example_doc_id)],
            embeddings=[self.example_embedding],
        )
        while True:
            search_response = self.vector._tablestore_client.search(
                table_name=self.vector._table_name,
                index_name=self.vector._index_name,
                search_query=tablestore.SearchQuery(query=tablestore.MatchAllQuery(), get_total_count=True, limit=0),
                columns_to_get=tablestore.ColumnsToGet(return_type=tablestore.ColumnReturnType.ALL_FROM_INDEX),
            )
            if search_response.total_count == 1:
                break

    def search_by_vector(self):
        super().search_by_vector()
        docs = self.vector.search_by_vector(self.example_embedding, document_ids_filter=[self.example_doc_id])
        assert len(docs) == 1
        assert docs[0].metadata["doc_id"] == self.example_doc_id
        assert docs[0].metadata["score"] > 0

        docs = self.vector.search_by_vector(self.example_embedding, document_ids_filter=[str(uuid.uuid4())])
        assert len(docs) == 0

    def search_by_full_text(self):
        super().search_by_full_text()
        docs = self.vector.search_by_full_text(get_example_text(), document_ids_filter=[self.example_doc_id])
        assert len(docs) == 1
        assert docs[0].metadata["doc_id"] == self.example_doc_id
        if self.vector._config.normalize_full_text_bm25_score:
            assert docs[0].metadata["score"] == approx(0.1214, abs=1e-3)
        else:
            assert docs[0].metadata.get("score") is None

        # return none if normalize_full_text_score=true and score_threshold > 0
        docs = self.vector.search_by_full_text(
            get_example_text(), document_ids_filter=[self.example_doc_id], score_threshold=0.5
        )
        if self.vector._config.normalize_full_text_bm25_score:
            assert len(docs) == 0
        else:
            assert len(docs) == 1
            assert docs[0].metadata["doc_id"] == self.example_doc_id
            assert docs[0].metadata.get("score") is None

        docs = self.vector.search_by_full_text(get_example_text(), document_ids_filter=[str(uuid.uuid4())])
        assert len(docs) == 0

    def run_all_tests(self):
        try:
            self.vector.delete()
        except Exception:
            pass

        return super().run_all_tests()


def test_tablestore_vector(setup_mock_redis):
    TableStoreVectorTest().run_all_tests()
    TableStoreVectorTest(normalize_full_text_score=True).run_all_tests()
    TableStoreVectorTest(normalize_full_text_score=False).run_all_tests()