You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

362 lines
9.9 KiB

package vector
import (
"encoding/json"
"fmt"
"godo/ai/server"
"godo/libs"
"godo/model"
"godo/office"
"log"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
)
func InitMonitorVector() {
// 确保数据库连接正常
// 确保数据库连接正常
db, err := model.Db.DB()
if err != nil {
log.Fatalf("Failed to get database connection: %v", err)
}
if err = db.Ping(); err != nil {
log.Fatalf("Failed to connect to database: %v", err)
}
list, err := GetVectorList()
if err != nil {
fmt.Println("GetVectorList error:", err)
return
}
if len(list) == 0 {
log.Println("no vector db found, creating a new one")
}
//log.Printf("init monitor:%v", list)
for _, v := range list {
MapFilePathMonitors[v.FilePath] = v.ID
}
go InitMonitor()
}
func HandlerCreateKnowledge(w http.ResponseWriter, r *http.Request) {
var req model.VecList
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
libs.ErrorMsg(w, "the chat request error:"+err.Error())
return
}
if req.FilePath == "" {
libs.ErrorMsg(w, "file path is empty")
return
}
basePath, err := libs.GetOsDir()
if err != nil {
libs.ErrorMsg(w, "get vector db path error:"+err.Error())
return
}
req.FilePath = filepath.Join(basePath, req.FilePath)
if !libs.PathExists(req.FilePath) {
libs.ErrorMsg(w, "the knowledge path is not exists")
return
}
fileInfo, err := os.Stat(req.FilePath)
if err != nil {
libs.ErrorMsg(w, "get vector db path error:"+err.Error())
return
}
if !fileInfo.IsDir() {
libs.ErrorMsg(w, "the knowledge path is not dir")
return
}
knowledgeFilePath := filepath.Join(req.FilePath, ".knowledge")
if libs.PathExists(knowledgeFilePath) {
libs.ErrorMsg(w, "the knowledgeId already exists")
return
}
id, err := CreateVector(req)
if err != nil {
libs.ErrorMsg(w, err.Error())
return
}
libs.SuccessMsg(w, id, "create vector success")
}
func HandlerAskKnowledge(w http.ResponseWriter, r *http.Request) {
var req model.AskRequest
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
libs.ErrorMsg(w, "the chat request error:"+err.Error())
return
}
if req.ID == 0 {
libs.ErrorMsg(w, "knowledgeId is empty")
return
}
var knowData model.VecList
if err := model.Db.First(&knowData, req.ID).Error; err != nil {
libs.ErrorMsg(w, err.Error())
return
}
var filterDocs []string
filterDocs = append(filterDocs, req.Input)
// 获取嵌入向量
resList, err := server.GetEmbeddings(knowData.Engine, knowData.EmbeddingModel, filterDocs)
if err != nil {
libs.ErrorMsg(w, err.Error())
return
}
res, err := model.AskDocument(req.ID, resList[0])
if err != nil {
libs.ErrorMsg(w, err.Error())
return
}
libs.SuccessMsg(w, res, "ask knowledge success")
}
func HandlerDelKnowledge(w http.ResponseWriter, r *http.Request) {
idStr := r.URL.Query().Get("id")
if idStr == "" {
libs.ErrorMsg(w, "knowledgeId is empty")
return
}
id, err := strconv.Atoi(idStr)
if err != nil {
libs.ErrorMsg(w, "knowledgeId is not number")
return
}
if id == 0 {
libs.ErrorMsg(w, "knowledgeId is not number")
return
}
if err := DeleteVector(uint(id)); err != nil {
libs.ErrorMsg(w, err.Error())
return
}
libs.SuccessMsg(w, nil, "delete knowledge success")
}
// CreateVector 创建一个新的 VectorList 记录
func CreateVector(data model.VecList) (uint, error) {
if data.FilePath == "" {
return 0, fmt.Errorf("file path is empty")
}
if data.Engine == "" {
return 0, fmt.Errorf("engine is empty")
}
if !libs.PathExists(data.FilePath) {
return 0, fmt.Errorf("file path does not exist")
}
if data.EmbeddingModel == "" {
return 0, fmt.Errorf("embedding model is empty")
}
if data.EmbedSize == 0 {
data.EmbedSize = 768
}
// Create the new VectorList
result := model.Db.Create(&data)
if result.Error != nil {
return 0, fmt.Errorf("failed to create vector list: %w", result.Error)
}
// 创建 .knowledge 文件并写入 knowledgeId
knowledgeFilePath := filepath.Join(data.FilePath, ".knowledge")
err := os.WriteFile(knowledgeFilePath, []byte(fmt.Sprintf("%d", data.ID)), 0644)
if err != nil {
return 0, fmt.Errorf("failed to write knowledgeId to .knowledge file: %w", err)
}
// // 等待 AddWatchFolder 完成
go AddWatchFolder(data.FilePath, data.ID, func() {
go office.SetDocument(data.FilePath, data.ID)
})
return data.ID, nil
}
// DeleteVector 删除指定id的 VectorList 记录
func DeleteVector(id uint) error {
var vectorList model.VecList
if err := model.Db.First(&vectorList, id).Error; err != nil {
return fmt.Errorf("failed to find vector list: %w", err)
}
// Delete .knowledge file
knowledgeFilePath := filepath.Join(vectorList.FilePath, ".knowledge")
if err := os.Remove(knowledgeFilePath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to delete .knowledge file: %w", err)
}
// Delete all .godoos. files in the directory and its subdirectories
if err := deleteGodoosFiles(vectorList.FilePath); err != nil {
return fmt.Errorf("failed to delete .godoos. files: %w", err)
}
//delete(MapFilePathMonitors, vectorList.FilePath)
RemoveWatchFolder(vectorList.FilePath)
return model.Db.Delete(&model.VecList{}, id).Error
}
func DeleteVectorFile(id uint, filePath string) error {
var vectorList model.VecList
if err := model.Db.First(&vectorList, id).Error; err != nil {
return fmt.Errorf("failed to find vector list: %w", err)
}
// Delete file in database
if err := model.Deletedocument(id, filePath); err != nil {
return fmt.Errorf("failed to delete .godoos. files: %w", err)
}
newFileName := fmt.Sprintf(".godoos.%d.%s.json", id, filepath.Base(filePath))
newFilePath := filepath.Join(filepath.Dir(filePath), newFileName)
if libs.PathExists(newFilePath) {
if err := os.Remove(newFilePath); err != nil {
return fmt.Errorf("failed to delete .godoos. files: %w", err)
}
}
return model.Db.Delete(&model.VecList{}, id).Error
}
func deleteGodoosFiles(dir string) error {
return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && filepath.Base(path)[:7] == ".godoos." {
if err := os.Remove(path); err != nil {
return err
}
}
return nil
})
}
// RenameVectorDb 更改指定名称的 VectorList 的数据库名称
func RenameVectorDb(oldName string, newName string) error {
basePath, err := libs.GetOsDir()
if err != nil {
return fmt.Errorf("failed to find old vector list: %w", err)
}
// 获取旧的 VectorList 记录
var oldList model.VecList
oldPath := filepath.Join(basePath, oldName)
if err := model.Db.Where("file_path = ?", oldPath).First(&oldList).Error; err != nil {
return fmt.Errorf("failed to find old vector list: %w", err)
}
// 更新 VectorList 记录中的 FilePath
newPath := filepath.Join(basePath, newName)
if err := model.Db.Model(&model.VecList{}).Where("id = ?", oldList.ID).Update("file_path", newPath).Error; err != nil {
return fmt.Errorf("failed to update vector list: %w", err)
}
// Update MapFilePathMonitors
//delete(MapFilePathMonitors, oldPath)
go RemoveWatchFolder(oldPath)
//MapFilePathMonitors[newPath] = oldList.ID
go AddWatchFolder(newPath, oldList.ID, nil)
return nil
}
func GetVectorList() ([]model.VecList, error) {
var vectorList []model.VecList
if err := model.Db.Find(&vectorList).Error; err != nil {
return nil, fmt.Errorf("failed to get vector list: %w", err)
}
return vectorList, nil
}
func GetVector(id uint) (model.VecList, error) {
var vectorList model.VecList
if err := model.Db.First(&vectorList, id).Error; err != nil {
return vectorList, fmt.Errorf("failed to get vector: %w", err)
}
return vectorList, nil
}
// handleGodoosFile 处理 .godoos 文件
func handleGodoosFile(filePath string, knowledgeId uint) error {
//log.Printf("========Handling .godoos file: %s", filePath)
baseName := filepath.Base(filePath)
// 检查文件后缀是否为 .exe
if strings.HasSuffix(baseName, ".exe") {
log.Printf("Skipping .exe file: %s", filePath)
return nil
}
// 检查是否为 .godoos 文件
if strings.HasPrefix(baseName, ".godoos.") {
// 去掉 .godoos. 前缀和 .json 后缀
// fileName := strings.TrimSuffix(strings.TrimPrefix(baseName, ".godoos."), ".json")
// //提取实际文件名部分
// actualFileName := extractFileName(fileName)
// 读取文件内容
content, err := os.ReadFile(filePath)
if err != nil {
return err
}
// 解析 JSON 内容
var doc office.Document
err = json.Unmarshal(content, &doc)
if err != nil {
return err
}
// 检查 Split 是否为空
if len(doc.Split) == 0 {
return fmt.Errorf("invalid .godoos file: %s", filePath)
}
// 获取向量数据
knowData, err := GetVector(knowledgeId)
if err != nil {
return err
}
basePath, err := libs.GetOsDir()
if err != nil {
return err
}
// 拼接文件名和内容
// var filterDocs []string
// for _, res := range doc.Split {
// filterDocs = append(filterDocs, fmt.Sprintf("%s %s", actualFileName, res))
// }
// 获取嵌入向量
resList, err := server.GetEmbeddings(knowData.Engine, knowData.EmbeddingModel, doc.Split)
if err != nil {
return err
}
// 检查嵌入向量长度是否匹配
if len(resList) != len(doc.Split) {
return fmt.Errorf("invalid file len: %s, expected %d embeddings, got %d", filePath, len(doc.Split), len(resList))
}
// 创建 VecDoc 列表
var vectordocs []model.VecDoc
for _, res := range doc.Split {
//log.Printf("Adding document: %s", res)
vectordoc := model.VecDoc{
Content: res,
FilePath: strings.TrimPrefix(doc.RePath, basePath),
FileName: doc.Title,
ListID: knowledgeId,
}
vectordocs = append(vectordocs, vectordoc)
}
// 添加文档
err = model.Adddocument(knowledgeId, vectordocs, resList)
return err
} else {
// 处理非 .godoos 文件
if baseName[:1] != "." {
office.ProcessFile(filePath, knowledgeId)
}
return nil
}
}
// func extractFileName(fileName string) string {
// // 假设文件名格式为:21.GodoOS企业版介绍
// parts := strings.SplitN(fileName, ".", 3)
// if len(parts) < 2 {
// return fileName
// }
// return parts[1]
// }