mirror of https://gitee.com/godoos/godoos.git
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
362 lines
9.9 KiB
362 lines
9.9 KiB
package vector
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"godo/ai/server"
|
|
"godo/libs"
|
|
"godo/model"
|
|
"godo/office"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
func InitMonitorVector() {
|
|
// 确保数据库连接正常
|
|
// 确保数据库连接正常
|
|
db, err := model.Db.DB()
|
|
if err != nil {
|
|
log.Fatalf("Failed to get database connection: %v", err)
|
|
}
|
|
if err = db.Ping(); err != nil {
|
|
log.Fatalf("Failed to connect to database: %v", err)
|
|
}
|
|
list, err := GetVectorList()
|
|
if err != nil {
|
|
fmt.Println("GetVectorList error:", err)
|
|
return
|
|
}
|
|
if len(list) == 0 {
|
|
log.Println("no vector db found, creating a new one")
|
|
}
|
|
//log.Printf("init monitor:%v", list)
|
|
for _, v := range list {
|
|
MapFilePathMonitors[v.FilePath] = v.ID
|
|
}
|
|
go InitMonitor()
|
|
}
|
|
func HandlerCreateKnowledge(w http.ResponseWriter, r *http.Request) {
|
|
var req model.VecList
|
|
err := json.NewDecoder(r.Body).Decode(&req)
|
|
if err != nil {
|
|
libs.ErrorMsg(w, "the chat request error:"+err.Error())
|
|
return
|
|
}
|
|
if req.FilePath == "" {
|
|
libs.ErrorMsg(w, "file path is empty")
|
|
return
|
|
}
|
|
basePath, err := libs.GetOsDir()
|
|
if err != nil {
|
|
libs.ErrorMsg(w, "get vector db path error:"+err.Error())
|
|
return
|
|
}
|
|
req.FilePath = filepath.Join(basePath, req.FilePath)
|
|
if !libs.PathExists(req.FilePath) {
|
|
libs.ErrorMsg(w, "the knowledge path is not exists")
|
|
return
|
|
}
|
|
fileInfo, err := os.Stat(req.FilePath)
|
|
if err != nil {
|
|
libs.ErrorMsg(w, "get vector db path error:"+err.Error())
|
|
return
|
|
}
|
|
if !fileInfo.IsDir() {
|
|
libs.ErrorMsg(w, "the knowledge path is not dir")
|
|
return
|
|
}
|
|
knowledgeFilePath := filepath.Join(req.FilePath, ".knowledge")
|
|
if libs.PathExists(knowledgeFilePath) {
|
|
libs.ErrorMsg(w, "the knowledgeId already exists")
|
|
return
|
|
}
|
|
id, err := CreateVector(req)
|
|
if err != nil {
|
|
libs.ErrorMsg(w, err.Error())
|
|
return
|
|
}
|
|
libs.SuccessMsg(w, id, "create vector success")
|
|
}
|
|
func HandlerAskKnowledge(w http.ResponseWriter, r *http.Request) {
|
|
var req model.AskRequest
|
|
err := json.NewDecoder(r.Body).Decode(&req)
|
|
if err != nil {
|
|
libs.ErrorMsg(w, "the chat request error:"+err.Error())
|
|
return
|
|
}
|
|
if req.ID == 0 {
|
|
libs.ErrorMsg(w, "knowledgeId is empty")
|
|
return
|
|
}
|
|
var knowData model.VecList
|
|
if err := model.Db.First(&knowData, req.ID).Error; err != nil {
|
|
libs.ErrorMsg(w, err.Error())
|
|
return
|
|
}
|
|
var filterDocs []string
|
|
filterDocs = append(filterDocs, req.Input)
|
|
// 获取嵌入向量
|
|
resList, err := server.GetEmbeddings(knowData.Engine, knowData.EmbeddingModel, filterDocs)
|
|
if err != nil {
|
|
libs.ErrorMsg(w, err.Error())
|
|
return
|
|
}
|
|
res, err := model.AskDocument(req.ID, resList[0])
|
|
if err != nil {
|
|
libs.ErrorMsg(w, err.Error())
|
|
return
|
|
}
|
|
libs.SuccessMsg(w, res, "ask knowledge success")
|
|
|
|
}
|
|
func HandlerDelKnowledge(w http.ResponseWriter, r *http.Request) {
|
|
idStr := r.URL.Query().Get("id")
|
|
if idStr == "" {
|
|
libs.ErrorMsg(w, "knowledgeId is empty")
|
|
return
|
|
}
|
|
id, err := strconv.Atoi(idStr)
|
|
if err != nil {
|
|
libs.ErrorMsg(w, "knowledgeId is not number")
|
|
return
|
|
}
|
|
if id == 0 {
|
|
libs.ErrorMsg(w, "knowledgeId is not number")
|
|
return
|
|
}
|
|
if err := DeleteVector(uint(id)); err != nil {
|
|
libs.ErrorMsg(w, err.Error())
|
|
return
|
|
}
|
|
libs.SuccessMsg(w, nil, "delete knowledge success")
|
|
}
|
|
|
|
// CreateVector 创建一个新的 VectorList 记录
|
|
func CreateVector(data model.VecList) (uint, error) {
|
|
if data.FilePath == "" {
|
|
return 0, fmt.Errorf("file path is empty")
|
|
}
|
|
if data.Engine == "" {
|
|
return 0, fmt.Errorf("engine is empty")
|
|
}
|
|
|
|
if !libs.PathExists(data.FilePath) {
|
|
return 0, fmt.Errorf("file path does not exist")
|
|
}
|
|
if data.EmbeddingModel == "" {
|
|
return 0, fmt.Errorf("embedding model is empty")
|
|
}
|
|
if data.EmbedSize == 0 {
|
|
data.EmbedSize = 768
|
|
}
|
|
// Create the new VectorList
|
|
result := model.Db.Create(&data)
|
|
if result.Error != nil {
|
|
return 0, fmt.Errorf("failed to create vector list: %w", result.Error)
|
|
}
|
|
// 创建 .knowledge 文件并写入 knowledgeId
|
|
knowledgeFilePath := filepath.Join(data.FilePath, ".knowledge")
|
|
err := os.WriteFile(knowledgeFilePath, []byte(fmt.Sprintf("%d", data.ID)), 0644)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to write knowledgeId to .knowledge file: %w", err)
|
|
}
|
|
|
|
// // 等待 AddWatchFolder 完成
|
|
go AddWatchFolder(data.FilePath, data.ID, func() {
|
|
go office.SetDocument(data.FilePath, data.ID)
|
|
})
|
|
|
|
return data.ID, nil
|
|
}
|
|
|
|
// DeleteVector 删除指定id的 VectorList 记录
|
|
func DeleteVector(id uint) error {
|
|
var vectorList model.VecList
|
|
if err := model.Db.First(&vectorList, id).Error; err != nil {
|
|
return fmt.Errorf("failed to find vector list: %w", err)
|
|
}
|
|
// Delete .knowledge file
|
|
knowledgeFilePath := filepath.Join(vectorList.FilePath, ".knowledge")
|
|
if err := os.Remove(knowledgeFilePath); err != nil && !os.IsNotExist(err) {
|
|
return fmt.Errorf("failed to delete .knowledge file: %w", err)
|
|
}
|
|
// Delete all .godoos. files in the directory and its subdirectories
|
|
if err := deleteGodoosFiles(vectorList.FilePath); err != nil {
|
|
return fmt.Errorf("failed to delete .godoos. files: %w", err)
|
|
}
|
|
//delete(MapFilePathMonitors, vectorList.FilePath)
|
|
RemoveWatchFolder(vectorList.FilePath)
|
|
return model.Db.Delete(&model.VecList{}, id).Error
|
|
}
|
|
|
|
func DeleteVectorFile(id uint, filePath string) error {
|
|
var vectorList model.VecList
|
|
if err := model.Db.First(&vectorList, id).Error; err != nil {
|
|
return fmt.Errorf("failed to find vector list: %w", err)
|
|
}
|
|
// Delete file in database
|
|
if err := model.Deletedocument(id, filePath); err != nil {
|
|
return fmt.Errorf("failed to delete .godoos. files: %w", err)
|
|
}
|
|
newFileName := fmt.Sprintf(".godoos.%d.%s.json", id, filepath.Base(filePath))
|
|
newFilePath := filepath.Join(filepath.Dir(filePath), newFileName)
|
|
if libs.PathExists(newFilePath) {
|
|
if err := os.Remove(newFilePath); err != nil {
|
|
return fmt.Errorf("failed to delete .godoos. files: %w", err)
|
|
}
|
|
}
|
|
return model.Db.Delete(&model.VecList{}, id).Error
|
|
}
|
|
func deleteGodoosFiles(dir string) error {
|
|
return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !info.IsDir() && filepath.Base(path)[:7] == ".godoos." {
|
|
if err := os.Remove(path); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// RenameVectorDb 更改指定名称的 VectorList 的数据库名称
|
|
func RenameVectorDb(oldName string, newName string) error {
|
|
basePath, err := libs.GetOsDir()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to find old vector list: %w", err)
|
|
}
|
|
|
|
// 获取旧的 VectorList 记录
|
|
var oldList model.VecList
|
|
oldPath := filepath.Join(basePath, oldName)
|
|
if err := model.Db.Where("file_path = ?", oldPath).First(&oldList).Error; err != nil {
|
|
return fmt.Errorf("failed to find old vector list: %w", err)
|
|
}
|
|
|
|
// 更新 VectorList 记录中的 FilePath
|
|
newPath := filepath.Join(basePath, newName)
|
|
if err := model.Db.Model(&model.VecList{}).Where("id = ?", oldList.ID).Update("file_path", newPath).Error; err != nil {
|
|
return fmt.Errorf("failed to update vector list: %w", err)
|
|
}
|
|
// Update MapFilePathMonitors
|
|
//delete(MapFilePathMonitors, oldPath)
|
|
go RemoveWatchFolder(oldPath)
|
|
//MapFilePathMonitors[newPath] = oldList.ID
|
|
go AddWatchFolder(newPath, oldList.ID, nil)
|
|
return nil
|
|
}
|
|
|
|
func GetVectorList() ([]model.VecList, error) {
|
|
var vectorList []model.VecList
|
|
if err := model.Db.Find(&vectorList).Error; err != nil {
|
|
return nil, fmt.Errorf("failed to get vector list: %w", err)
|
|
}
|
|
return vectorList, nil
|
|
}
|
|
|
|
func GetVector(id uint) (model.VecList, error) {
|
|
var vectorList model.VecList
|
|
if err := model.Db.First(&vectorList, id).Error; err != nil {
|
|
return vectorList, fmt.Errorf("failed to get vector: %w", err)
|
|
}
|
|
return vectorList, nil
|
|
}
|
|
|
|
// handleGodoosFile 处理 .godoos 文件
|
|
func handleGodoosFile(filePath string, knowledgeId uint) error {
|
|
//log.Printf("========Handling .godoos file: %s", filePath)
|
|
baseName := filepath.Base(filePath)
|
|
// 检查文件后缀是否为 .exe
|
|
if strings.HasSuffix(baseName, ".exe") {
|
|
log.Printf("Skipping .exe file: %s", filePath)
|
|
return nil
|
|
}
|
|
// 检查是否为 .godoos 文件
|
|
if strings.HasPrefix(baseName, ".godoos.") {
|
|
// 去掉 .godoos. 前缀和 .json 后缀
|
|
// fileName := strings.TrimSuffix(strings.TrimPrefix(baseName, ".godoos."), ".json")
|
|
// //提取实际文件名部分
|
|
// actualFileName := extractFileName(fileName)
|
|
|
|
// 读取文件内容
|
|
content, err := os.ReadFile(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// 解析 JSON 内容
|
|
var doc office.Document
|
|
err = json.Unmarshal(content, &doc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// 检查 Split 是否为空
|
|
if len(doc.Split) == 0 {
|
|
return fmt.Errorf("invalid .godoos file: %s", filePath)
|
|
}
|
|
|
|
// 获取向量数据
|
|
knowData, err := GetVector(knowledgeId)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
basePath, err := libs.GetOsDir()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// 拼接文件名和内容
|
|
// var filterDocs []string
|
|
// for _, res := range doc.Split {
|
|
// filterDocs = append(filterDocs, fmt.Sprintf("%s %s", actualFileName, res))
|
|
// }
|
|
// 获取嵌入向量
|
|
resList, err := server.GetEmbeddings(knowData.Engine, knowData.EmbeddingModel, doc.Split)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// 检查嵌入向量长度是否匹配
|
|
if len(resList) != len(doc.Split) {
|
|
return fmt.Errorf("invalid file len: %s, expected %d embeddings, got %d", filePath, len(doc.Split), len(resList))
|
|
}
|
|
|
|
// 创建 VecDoc 列表
|
|
var vectordocs []model.VecDoc
|
|
for _, res := range doc.Split {
|
|
//log.Printf("Adding document: %s", res)
|
|
vectordoc := model.VecDoc{
|
|
Content: res,
|
|
FilePath: strings.TrimPrefix(doc.RePath, basePath),
|
|
FileName: doc.Title,
|
|
ListID: knowledgeId,
|
|
}
|
|
vectordocs = append(vectordocs, vectordoc)
|
|
}
|
|
|
|
// 添加文档
|
|
err = model.Adddocument(knowledgeId, vectordocs, resList)
|
|
return err
|
|
} else {
|
|
// 处理非 .godoos 文件
|
|
if baseName[:1] != "." {
|
|
office.ProcessFile(filePath, knowledgeId)
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// func extractFileName(fileName string) string {
|
|
// // 假设文件名格式为:21.GodoOS企业版介绍
|
|
// parts := strings.SplitN(fileName, ".", 3)
|
|
// if len(parts) < 2 {
|
|
// return fileName
|
|
// }
|
|
// return parts[1]
|
|
// }
|
|
|