mirror of https://gitee.com/godoos/godoos.git
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
181 lines
4.2 KiB
181 lines
4.2 KiB
package vector
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"godo/ai/server"
|
|
"godo/libs"
|
|
"godo/office"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/fsnotify/fsnotify"
|
|
)
|
|
|
|
var MapFilePathMonitors = map[string]uint{}
|
|
|
|
func FolderMonitor() {
|
|
basePath, err := libs.GetOsDir()
|
|
if err != nil {
|
|
log.Printf("Error getting base path: %s", err.Error())
|
|
return
|
|
}
|
|
watcher, err := fsnotify.NewWatcher()
|
|
if err != nil {
|
|
log.Printf("Error creating watcher: %s", err.Error())
|
|
return
|
|
}
|
|
defer watcher.Close()
|
|
|
|
// 递归添加所有子目录
|
|
addRecursive(basePath, watcher)
|
|
|
|
// Start listening for events.
|
|
go func() {
|
|
for {
|
|
select {
|
|
case event, ok := <-watcher.Events:
|
|
if !ok {
|
|
log.Println("error:", err)
|
|
return
|
|
}
|
|
//log.Println("event:", event)
|
|
filePath := event.Name
|
|
result, knowledgeId := shouldProcess(filePath)
|
|
//log.Printf("result:%d,knowledgeId:%d", result, knowledgeId)
|
|
if result > 0 {
|
|
info, err := os.Stat(filePath)
|
|
if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) {
|
|
log.Println("modified file:", filePath)
|
|
if !info.IsDir() {
|
|
handleGodoosFile(filePath, knowledgeId)
|
|
}
|
|
}
|
|
if event.Has(fsnotify.Create) || event.Has(fsnotify.Rename) {
|
|
// 处理创建或重命名事件,添加新目录
|
|
if err == nil && info.IsDir() {
|
|
addRecursive(filePath, watcher)
|
|
}
|
|
}
|
|
if event.Has(fsnotify.Remove) {
|
|
// 处理删除事件,移除目录
|
|
if err == nil && info.IsDir() {
|
|
watcher.Remove(filePath)
|
|
}
|
|
}
|
|
}
|
|
case err, ok := <-watcher.Errors:
|
|
if !ok {
|
|
return
|
|
}
|
|
log.Println("error:", err)
|
|
}
|
|
}
|
|
}()
|
|
|
|
// Add a path.
|
|
err = watcher.Add(basePath)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// Block main goroutine forever.
|
|
<-make(chan struct{})
|
|
}
|
|
|
|
func shouldProcess(filePath string) (int, uint) {
|
|
// 规范化路径
|
|
filePath = filepath.Clean(filePath)
|
|
|
|
// 检查文件路径是否在 MapFilePathMonitors 中
|
|
for path, id := range MapFilePathMonitors {
|
|
if id < 1 {
|
|
return 0, 0
|
|
}
|
|
path = filepath.Clean(path)
|
|
if filePath == path {
|
|
return 1, id // 完全相等
|
|
}
|
|
if strings.HasPrefix(filePath, path+string(filepath.Separator)) {
|
|
return 2, id // 包含
|
|
}
|
|
}
|
|
return 0, 0 // 不存在
|
|
}
|
|
|
|
func addRecursive(path string, watcher *fsnotify.Watcher) {
|
|
err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
log.Printf("Error walking path %s: %v", path, err)
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
result, _ := shouldProcess(path)
|
|
if result > 0 {
|
|
if err := watcher.Add(path); err != nil {
|
|
log.Printf("Error adding path %s to watcher: %v", path, err)
|
|
return err
|
|
}
|
|
log.Printf("Added path %s to watcher", path)
|
|
}
|
|
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
log.Printf("Error adding recursive paths: %v", err)
|
|
}
|
|
}
|
|
|
|
func handleGodoosFile(filePath string, knowledgeId uint) error {
|
|
log.Printf("========Handling .godoos file: %s", filePath)
|
|
baseName := filepath.Base(filePath)
|
|
if baseName[:8] != ".godoos." {
|
|
if baseName[:1] != "." {
|
|
office.ProcessFile(filePath, knowledgeId)
|
|
}
|
|
return nil
|
|
}
|
|
var doc office.Document
|
|
content, err := os.ReadFile(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = json.Unmarshal(content, &doc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(doc.Split) == 0 {
|
|
return fmt.Errorf("invalid .godoos file: %s", filePath)
|
|
}
|
|
knowData, err := GetVector(knowledgeId)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resList, err := server.GetEmbeddings(knowData.Engine, knowData.EmbeddingModel, doc.Split)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(resList) != len(doc.Split) {
|
|
return fmt.Errorf("invalid file len: %s, expected %d embeddings, got %d", filePath, len(doc.Split), len(resList))
|
|
}
|
|
// var vectordocs []model.Vectordoc
|
|
// for i, res := range resList {
|
|
// //log.Printf("res: %v", res)
|
|
// vectordoc := model.Vectordoc{
|
|
// Content: doc.Split[i],
|
|
// Embed: res,
|
|
// FilePath: filePath,
|
|
// KnowledgeID: knowledgeId,
|
|
// Pos: fmt.Sprintf("%d", i),
|
|
// }
|
|
// vectordocs = append(vectordocs, vectordoc)
|
|
// }
|
|
// result := vectorListDb.Create(&vectordocs)
|
|
// if result.Error != nil {
|
|
// return result.Error
|
|
// }
|
|
return nil
|
|
}
|
|
|