dify-1.8.1/web/service/datasets.ts

import type { Fetcher } from 'swr'
import qs from 'qs'
import { del, get, patch, post, put } from './base'
import type {
  CreateDocumentReq,
  DataSet,
  DataSetListResponse,
  ErrorDocsResponse,
  ExternalAPIDeleteResponse,
  ExternalAPIItem,
  ExternalAPIListResponse,
  ExternalAPIUsage,
  ExternalKnowledgeBaseHitTestingResponse,
  ExternalKnowledgeItem,
  FetchDatasetsParams,
  FileIndexingEstimateResponse,
  HitTestingRecordsResponse,
  HitTestingResponse,
  IndexingEstimateParams,
  IndexingEstimateResponse,
  IndexingStatusBatchResponse,
  IndexingStatusResponse,
  ProcessRuleResponse,
  RelatedAppResponse,
  createDocumentResponse,
} from '@/models/datasets'
import type { CreateKnowledgeBaseReq } from '@/app/components/datasets/external-knowledge-base/create/declarations'
import type { CreateExternalAPIReq } from '@/app/components/datasets/external-api/declarations'
import type { CommonResponse, DataSourceNotionWorkspace } from '@/models/common'
import { DataSourceProvider } from '@/models/common'
import type {
  ApiKeysListResponse,
  CreateApiKeyResponse,
} from '@/models/app'
import type { RetrievalConfig } from '@/types/app'

// apis for documents in a dataset

type CommonDocReq = {
  datasetId: string
  documentId: string
}

type BatchReq = {
  datasetId: string
  batchId: string
}

export type SortType = 'created_at' | 'hit_count' | '-created_at' | '-hit_count'

export type MetadataType = 'all' | 'only' | 'without'

export const fetchDatasetDetail: Fetcher<DataSet, string> = (datasetId: string) => {
  return get<DataSet>(`/datasets/${datasetId}`)
}

export const updateDatasetSetting: Fetcher<DataSet, {
  datasetId: string
  body: Partial<Pick<DataSet,
    'name' | 'description' | 'permission' | 'partial_member_list' | 'indexing_technique' | 'retrieval_model' | 'embedding_model' | 'embedding_model_provider'
  >>
}> = ({ datasetId, body }) => {
  return patch<DataSet>(`/datasets/${datasetId}`, { body })
}

export const fetchDatasetRelatedApps: Fetcher<RelatedAppResponse, string> = (datasetId: string) => {
  return get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`)
}

export const fetchDatasets: Fetcher<DataSetListResponse, FetchDatasetsParams> = ({ url, params }) => {
  const urlParams = qs.stringify(params, { indices: false })
  return get<DataSetListResponse>(`${url}?${urlParams}`)
}

export const createEmptyDataset: Fetcher<DataSet, { name: string }> = ({ name }) => {
  return post<DataSet>('/datasets', { body: { name } })
}

export const checkIsUsedInApp: Fetcher<{ is_using: boolean }, string> = (id) => {
  return get<{ is_using: boolean }>(`/datasets/${id}/use-check`, {}, {
    silent: true,
  })
}

export const deleteDataset: Fetcher<DataSet, string> = (datasetID) => {
  return del<DataSet>(`/datasets/${datasetID}`)
}

export const fetchExternalAPIList: Fetcher<ExternalAPIListResponse, { url: string }> = ({ url }) => {
  return get<ExternalAPIListResponse>(url)
}

export const fetchExternalAPI: Fetcher<ExternalAPIItem, { apiTemplateId: string }> = ({ apiTemplateId }) => {
  return get<ExternalAPIItem>(`/datasets/external-knowledge-api/${apiTemplateId}`)
}

export const updateExternalAPI: Fetcher<ExternalAPIItem, { apiTemplateId: string; body: ExternalAPIItem }> = ({ apiTemplateId, body }) => {
  return patch<ExternalAPIItem>(`/datasets/external-knowledge-api/${apiTemplateId}`, { body })
}

export const deleteExternalAPI: Fetcher<ExternalAPIDeleteResponse, { apiTemplateId: string }> = ({ apiTemplateId }) => {
  return del<ExternalAPIDeleteResponse>(`/datasets/external-knowledge-api/${apiTemplateId}`)
}

export const checkUsageExternalAPI: Fetcher<ExternalAPIUsage, { apiTemplateId: string }> = ({ apiTemplateId }) => {
  return get<ExternalAPIUsage>(`/datasets/external-knowledge-api/${apiTemplateId}/use-check`)
}

export const createExternalAPI: Fetcher<ExternalAPIItem, { body: CreateExternalAPIReq }> = ({ body }) => {
  return post<ExternalAPIItem>('/datasets/external-knowledge-api', { body })
}

export const createExternalKnowledgeBase: Fetcher<ExternalKnowledgeItem, { body: CreateKnowledgeBaseReq }> = ({ body }) => {
  return post<ExternalKnowledgeItem>('/datasets/external', { body })
}

export const fetchDefaultProcessRule: Fetcher<ProcessRuleResponse, { url: string }> = ({ url }) => {
  return get<ProcessRuleResponse>(url)
}
export const fetchProcessRule: Fetcher<ProcessRuleResponse, { params: { documentId: string } }> = ({ params: { documentId } }) => {
  return get<ProcessRuleResponse>('/datasets/process-rule', { params: { document_id: documentId } })
}

export const createFirstDocument: Fetcher<createDocumentResponse, { body: CreateDocumentReq }> = ({ body }) => {
  return post<createDocumentResponse>('/datasets/init', { body })
}

export const createDocument: Fetcher<createDocumentResponse, { datasetId: string; body: CreateDocumentReq }> = ({ datasetId, body }) => {
  return post<createDocumentResponse>(`/datasets/${datasetId}/documents`, { body })
}

export const fetchIndexingEstimate: Fetcher<IndexingEstimateResponse, CommonDocReq> = ({ datasetId, documentId }) => {
  return get<IndexingEstimateResponse>(`/datasets/${datasetId}/documents/${documentId}/indexing-estimate`, {})
}
export const fetchIndexingEstimateBatch: Fetcher<IndexingEstimateResponse, BatchReq> = ({ datasetId, batchId }) => {
  return get<IndexingEstimateResponse>(`/datasets/${datasetId}/batch/${batchId}/indexing-estimate`, {})
}

export const fetchIndexingStatus: Fetcher<IndexingStatusResponse, CommonDocReq> = ({ datasetId, documentId }) => {
  return get<IndexingStatusResponse>(`/datasets/${datasetId}/documents/${documentId}/indexing-status`, {})
}

export const fetchIndexingStatusBatch: Fetcher<IndexingStatusBatchResponse, BatchReq> = ({ datasetId, batchId }) => {
  return get<IndexingStatusBatchResponse>(`/datasets/${datasetId}/batch/${batchId}/indexing-status`, {})
}

export const renameDocumentName: Fetcher<CommonResponse, CommonDocReq & { name: string }> = ({ datasetId, documentId, name }) => {
  return post<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/rename`, {
    body: { name },
  })
}

export const pauseDocIndexing: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
  return patch<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/processing/pause`)
}

export const resumeDocIndexing: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
  return patch<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/processing/resume`)
}

export const preImportNotionPages: Fetcher<{ notion_info: DataSourceNotionWorkspace[] }, { url: string; datasetId?: string }> = ({ url, datasetId }) => {
  return get<{ notion_info: DataSourceNotionWorkspace[] }>(url, { params: { dataset_id: datasetId } })
}

export const modifyDocMetadata: Fetcher<CommonResponse, CommonDocReq & { body: { doc_type: string; doc_metadata: Record<string, any> } }> = ({ datasetId, documentId, body }) => {
  return put<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/metadata`, { body })
}

// hit testing
export const hitTesting: Fetcher<HitTestingResponse, { datasetId: string; queryText: string; retrieval_model: RetrievalConfig }> = ({ datasetId, queryText, retrieval_model }) => {
  return post<HitTestingResponse>(`/datasets/${datasetId}/hit-testing`, { body: { query: queryText, retrieval_model } })
}

export const externalKnowledgeBaseHitTesting: Fetcher<ExternalKnowledgeBaseHitTestingResponse, { datasetId: string; query: string; external_retrieval_model: { top_k: number; score_threshold: number; score_threshold_enabled: boolean } }> = ({ datasetId, query, external_retrieval_model }) => {
  return post<ExternalKnowledgeBaseHitTestingResponse>(`/datasets/${datasetId}/external-hit-testing`, { body: { query, external_retrieval_model } })
}

export const fetchTestingRecords: Fetcher<HitTestingRecordsResponse, { datasetId: string; params: { page: number; limit: number } }> = ({ datasetId, params }) => {
  return get<HitTestingRecordsResponse>(`/datasets/${datasetId}/queries`, { params })
}

export const fetchFileIndexingEstimate: Fetcher<FileIndexingEstimateResponse, IndexingEstimateParams> = (body: IndexingEstimateParams) => {
  return post<FileIndexingEstimateResponse>('/datasets/indexing-estimate', { body })
}

export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID: string; pageID: string; pageType: string }> = ({ workspaceID, pageID, pageType }) => {
  return get<{ content: string }>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`)
}

export const fetchApiKeysList: Fetcher<ApiKeysListResponse, { url: string; params: Record<string, any> }> = ({ url, params }) => {
  return get<ApiKeysListResponse>(url, params)
}

export const delApikey: Fetcher<CommonResponse, { url: string; params: Record<string, any> }> = ({ url, params }) => {
  return del<CommonResponse>(url, params)
}

export const createApikey: Fetcher<CreateApiKeyResponse, { url: string; body: Record<string, any> }> = ({ url, body }) => {
  return post<CreateApiKeyResponse>(url, body)
}

export const fetchDatasetApiBaseUrl: Fetcher<{ api_base_url: string }, string> = (url) => {
  return get<{ api_base_url: string }>(url)
}

export const fetchDataSources = () => {
  return get<CommonResponse>('api-key-auth/data-source')
}

export const createDataSourceApiKeyBinding: Fetcher<CommonResponse, Record<string, any>> = (body) => {
  return post<CommonResponse>('api-key-auth/data-source/binding', { body })
}

export const removeDataSourceApiKeyBinding: Fetcher<CommonResponse, string> = (id: string) => {
  return del<CommonResponse>(`api-key-auth/data-source/${id}`)
}

export const createFirecrawlTask: Fetcher<CommonResponse, Record<string, any>> = (body) => {
  return post<CommonResponse>('website/crawl', {
    body: {
      ...body,
      provider: DataSourceProvider.fireCrawl,
    },
  })
}

export const checkFirecrawlTaskStatus: Fetcher<CommonResponse, string> = (jobId: string) => {
  return get<CommonResponse>(`website/crawl/status/${jobId}`, {
    params: {
      provider: DataSourceProvider.fireCrawl,
    },
  }, {
    silent: true,
  })
}

export const createJinaReaderTask: Fetcher<CommonResponse, Record<string, any>> = (body) => {
  return post<CommonResponse>('website/crawl', {
    body: {
      ...body,
      provider: DataSourceProvider.jinaReader,
    },
  })
}

export const checkJinaReaderTaskStatus: Fetcher<CommonResponse, string> = (jobId: string) => {
  return get<CommonResponse>(`website/crawl/status/${jobId}`, {
    params: {
      provider: 'jinareader',
    },
  }, {
    silent: true,
  })
}

export const createWatercrawlTask: Fetcher<CommonResponse, Record<string, any>> = (body) => {
  return post<CommonResponse>('website/crawl', {
    body: {
      ...body,
      provider: DataSourceProvider.waterCrawl,
    },
  })
}

export const checkWatercrawlTaskStatus: Fetcher<CommonResponse, string> = (jobId: string) => {
  return get<CommonResponse>(`website/crawl/status/${jobId}`, {
    params: {
      provider: DataSourceProvider.waterCrawl,
    },
  }, {
    silent: true,
  })
}

type FileTypesRes = {
  allowed_extensions: string[]
}

export const fetchSupportFileTypes: Fetcher<FileTypesRes, { url: string }> = ({ url }) => {
  return get<FileTypesRes>(url)
}

export const getErrorDocs: Fetcher<ErrorDocsResponse, { datasetId: string }> = ({ datasetId }) => {
  return get<ErrorDocsResponse>(`/datasets/${datasetId}/error-docs`)
}

export const retryErrorDocs: Fetcher<CommonResponse, { datasetId: string; document_ids: string[] }> = ({ datasetId, document_ids }) => {
  return post<CommonResponse>(`/datasets/${datasetId}/retry`, { body: { document_ids } })
}

// ============ Segment Version Control APIs ============

export type SegmentVersion = {
  version_number: number
  segment_id: string
  document_id: string
  document_name: string
  dataset_id: string
  dataset_name: string
  content: string
  answer?: string
  keywords?: string[]
  enabled: boolean
  position: number
  word_count: number
  change_type: 'create' | 'update' | 'delete' | 'rollback'
  change_summary: string
  changed_fields: string[]
  created_by: string
  created_by_name: string
  created_at: string
  created_ip?: string
  content_hash: string
}

export type VersionInfo = {
  version_number: number
  created_at: string
  created_by_name: string
  change_type: string
}

export type VersionDiff = {
  changed: boolean
  old_value: any
  new_value: any
}

export type OperationLog = {
  id: string
  tenant_id: string
  account_id: string
  action: string
  content: {
    dataset_id: string
    resource_type: string
    resource_id: string
    action: string
    timestamp: string
    changes?: {
      old: any
      new: any
      changed_fields: string[]
    }
    metadata?: Record<string, any>
  }
  created_at: string
  created_ip: string
}

/**
 * Fetch segment version history
 */
export const fetchSegmentVersions = (
  datasetId: string,
  documentId: string,
  segmentId: string,
  params?: {
    page?: number
    per_page?: number
    include_content?: boolean
  },
) => {
  return get<{
    versions: SegmentVersion[]
    total: number
    page: number
    per_page: number
    pages: number
  }>(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/versions`, {
    params: params || {},
  })
}

/**
 * Fetch segment version detail
 */
export const fetchSegmentVersionDetail = (
  datasetId: string,
  documentId: string,
  segmentId: string,
  versionNumber: number,
) => {
  return get<SegmentVersion>(
    `/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/versions/${versionNumber}`,
  )
}

/**
 * Compare two segment versions
 */
export const compareSegmentVersions = (
  datasetId: string,
  documentId: string,
  segmentId: string,
  v1: number,
  v2: number,
) => {
  return get<{
    version_1: VersionInfo
    version_2: VersionInfo
    differences: Record<string, VersionDiff>
    has_changes: boolean
  }>(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/versions/compare`, {
    params: { v1, v2 },
  })
}

/**
 * Fetch segment version statistics
 */
export const fetchSegmentVersionStatistics = (
  datasetId: string,
  documentId: string,
  segmentId: string,
) => {
  return get<{
    total_versions: number
    first_version_at: string | null
    latest_version_at: string | null
    latest_version_number: number
    ttl_days: number
    max_versions: number
  }>(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}/versions/statistics`)
}

/**
 * Fetch dataset operation logs
 */
export const fetchDatasetOperationLogs = (
  datasetId: string,
  params?: {
    resource_type?: string
    resource_id?: string
    action?: string
    account_id?: string
    start_date?: string
    end_date?: string
    page?: number
    per_page?: number
  },
) => {
  return get<{
    logs: OperationLog[]
    total: number
    page: number
    per_page: number
    pages: number
  }>(`/datasets/${datasetId}/operation-logs`, {
    params: params || {},
  })
}

/**
 * Fetch document download permission configuration
 */
export const fetchDownloadPermission = (datasetId: string, documentId: string) => {
  return get<{
    is_restricted: boolean
    users: Array<{
      id: string
      name: string
      email: string
      has_permission: boolean
    }>
  }>(`/datasets/${datasetId}/documents/${documentId}/download-permission`)
}

/**
 * Update document download permission
 */
export const updateDownloadPermission = (
  datasetId: string,
  documentId: string,
  body: {
    is_restricted: boolean
    user_ids: string[]
  },
) => {
  return post<{ result: string }>(`/datasets/${datasetId}/documents/${documentId}/download-permission`, {
    body,
  })
}

/**
 * Fetch document download logs
 */
export const fetchDownloadLogs = (datasetId: string, documentId: string, limit = 100) => {
  return get<{
    logs: Array<{
      user_id: string
      user_name: string
      download_time: string
      ip: string
    }>
    total: number
    unique_users: number
  }>(`/datasets/${datasetId}/documents/${documentId}/download-logs`, {
    params: { limit },
  })
}