diff --git a/README.md b/README.md index 59978e7..37fba1f 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,9 @@ - 新增配置本地代理和远程代理,本地代理可实现本机ip映射外部域名,远程代理内嵌frpc设置后可实现内网用户外网域名访问。 - 修改锁屏机制,确保外网访问安全。 +- 支持本地聊天ai对话文件和联网搜索。 -## 🏭 第三阶段目标(十二月底发布) +## 🏭 第三阶段目标(一月底发布) 1. **文档处理与Markdown智能升级**:(已完成) - **AI续写**:借助先进的自然语言处理技术,让您的文档创作灵感不断,续写流畅无阻。 - **智能总结**:一键提取文档精髓,快速生成精炼总结,助力高效阅读与信息提炼。 diff --git a/frontend/src/components/builtin/FileList.vue b/frontend/src/components/builtin/FileList.vue index d6fd186..f9fe5ec 100644 --- a/frontend/src/components/builtin/FileList.vue +++ b/frontend/src/components/builtin/FileList.vue @@ -72,6 +72,7 @@ import { dealSystemName, t } from "@/i18n"; import { useChooseStore } from "@/stores/choose"; import { getSystemKey } from "@/system/config"; import { emitEvent, mountEvent } from "@/system/event"; +import { addKnowledge } from "@/hook/useAi"; import { basename, BrowserWindow, @@ -343,6 +344,26 @@ function handleRightClick( }, }); } + menuArr.push({ + label: "加入知识库", + click: () => { + //console.log(item.path) + addKnowledge(item.path).then((res:any) => { + console.log(res) + if(res.code != 0){ + new Notify({ + title: t("tips"), + content: res.message, + }); + }else{ + new Notify({ + title: t("tips"), + content: "添加成功", + }); + } + }) + }, + }); } if (choose.ifShow) { menuArr.push({ diff --git a/frontend/src/hook/useAi.ts b/frontend/src/hook/useAi.ts index eb3850f..53b68ae 100644 --- a/frontend/src/hook/useAi.ts +++ b/frontend/src/hook/useAi.ts @@ -61,4 +61,29 @@ export async function askAi(question: any, action: string) { } +} +export async function addKnowledge(path: string) { + const modelStore = useModelStore(); + const config = getSystemConfig(); + const model = await modelStore.getModel('embeddings') + if (!model) { + return { + code:-1, + message:'请先设置嵌入模型' + } + } + const apiUrl = config.aiUrl + '/ai/addknowledge' + const postMsg: any = { + engine: model.info.engine, + model: model.model, + file_path: path + }; + const complain = await fetchPost(apiUrl, JSON.stringify(postMsg)) + if (!complain.ok) { + return { + code:-1, + message:'请求失败' + } + } + return await complain.json() } \ No newline at end of file diff --git a/godo/ai/server/embedding.go b/godo/ai/server/embedding.go new file mode 100644 index 0000000..d396dcc --- /dev/null +++ b/godo/ai/server/embedding.go @@ -0,0 +1,75 @@ +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net/http" +) + +func GetEmbeddings(engine string, model string, text []string) ([][]float32, error) { + if engine == "ollama" { + res, err := getOllamaEmbedding(model, text) + if err != nil { + log.Printf("getOllamaEmbedding error: %v", err) + return nil, err + } + return res, nil + } + return nil, nil +} + +func getOllamaEmbedding(model string, text []string) ([][]float32, error) { + url := GetOllamaUrl() + "/api/embed" + //log.Printf("url: %s", url) + client := &http.Client{} + + // Prepare the request body. + reqBody, err := json.Marshal(map[string]interface{}{ + "model": model, + "input": text, + }) + //log.Printf("reqBody: %s", reqBody) + if err != nil { + return nil, fmt.Errorf("couldn't marshal request body: %w", err) + } + req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody)) + if err != nil { + return nil, fmt.Errorf("couldn't create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + // Send the request. + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("couldn't send request: %w", err) + } + defer resp.Body.Close() + + // Check the response status. + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("error response from the embedding API: %s", resp.Status) + } + + // Read and decode the response body. + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("couldn't read response body: %w", err) + } + var embeddingResponse struct { + Embeddings [][]float32 `json:"embeddings"` + } + err = json.Unmarshal(body, &embeddingResponse) + if err != nil { + return nil, fmt.Errorf("couldn't unmarshal response body: %w", err) + } + log.Printf("Embedding: %v", embeddingResponse.Embeddings) + + // Return the embeddings directly. + if len(embeddingResponse.Embeddings) == 0 { + return nil, fmt.Errorf("no embeddings returned from the API") + } + return embeddingResponse.Embeddings, nil +} diff --git a/godo/ai/vector/document.go b/godo/ai/vector/document.go index 48b77f8..29e5866 100644 --- a/godo/ai/vector/document.go +++ b/godo/ai/vector/document.go @@ -1,87 +1,178 @@ package vector import ( + "encoding/json" + "fmt" + "godo/ai/server" "godo/libs" "godo/office" "log" - "path" - "sync" + "os" + "path/filepath" + "strings" "github.com/fsnotify/fsnotify" ) -const numWorkers = 5 // 设置 worker 数量 +var MapFilePathMonitors = map[string]uint{} -func MonitorFolder(folderPath string) { - if !libs.PathExists(folderPath) { +func FolderMonitor() { + basePath, err := libs.GetOsDir() + if err != nil { + log.Printf("Error getting base path: %s", err.Error()) return } watcher, err := fsnotify.NewWatcher() if err != nil { - log.Fatalf("Failed to create watcher for folder %s: %v", folderPath, err) + log.Printf("Error creating watcher: %s", err.Error()) + return } defer watcher.Close() - fileQueue := make(chan string, 100) // 创建文件路径队列 - var wg sync.WaitGroup - - // 启动 worker goroutine - for i := 0; i < numWorkers; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for filePath := range fileQueue { - handleGodoosFile(filePath) - } - }() - } + // 递归添加所有子目录 + addRecursive(basePath, watcher) - done := make(chan bool) + // Start listening for events. go func() { for { select { case event, ok := <-watcher.Events: if !ok { + log.Println("error:", err) return } - log.Println("event:", event) - if event.Op&(fsnotify.Create|fsnotify.Write) != 0 { // 监听创建和修改事件 - baseName := path.Base(event.Name) - if baseName[:8] == ".godoos." { // 检查文件名是否以 .godoos 开头 - log.Printf("Detected .godoos file: %s", event.Name) - fileQueue <- event.Name // 将文件路径放入队列 - } else { - if baseName[:1] == "." { - return + //log.Println("event:", event) + filePath := event.Name + result, knowledgeId := shouldProcess(filePath) + //log.Printf("result:%d,knowledgeId:%d", result, knowledgeId) + if result > 0 { + info, err := os.Stat(filePath) + if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) { + log.Println("modified file:", filePath) + if !info.IsDir() { + handleGodoosFile(filePath, knowledgeId) + } + } + if event.Has(fsnotify.Create) || event.Has(fsnotify.Rename) { + // 处理创建或重命名事件,添加新目录 + if err == nil && info.IsDir() { + addRecursive(filePath, watcher) + } + } + if event.Has(fsnotify.Remove) { + // 处理删除事件,移除目录 + if err == nil && info.IsDir() { + watcher.Remove(filePath) } - office.ProcessFile(event.Name) } } case err, ok := <-watcher.Errors: if !ok { return } - log.Printf("Error watching folder %s: %v", folderPath, err) + log.Println("error:", err) } } }() - err = watcher.Add(folderPath) + // Add a path. + err = watcher.Add(basePath) if err != nil { - log.Fatalf("Failed to add folder %s to watcher: %v", folderPath, err) + log.Fatal(err) } - // 关闭文件队列 - go func() { - <-done - close(fileQueue) - wg.Wait() - }() + // Block main goroutine forever. + <-make(chan struct{}) +} + +func shouldProcess(filePath string) (int, uint) { + // 规范化路径 + filePath = filepath.Clean(filePath) + + // 检查文件路径是否在 MapFilePathMonitors 中 + for path, id := range MapFilePathMonitors { + if id < 1 { + return 0, 0 + } + path = filepath.Clean(path) + if filePath == path { + return 1, id // 完全相等 + } + if strings.HasPrefix(filePath, path+string(filepath.Separator)) { + return 2, id // 包含 + } + } + return 0, 0 // 不存在 +} + +func addRecursive(path string, watcher *fsnotify.Watcher) { + err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + log.Printf("Error walking path %s: %v", path, err) + return err + } + if info.IsDir() { + result, _ := shouldProcess(path) + if result > 0 { + if err := watcher.Add(path); err != nil { + log.Printf("Error adding path %s to watcher: %v", path, err) + return err + } + log.Printf("Added path %s to watcher", path) + } - <-done + } + return nil + }) + if err != nil { + log.Printf("Error adding recursive paths: %v", err) + } } -func handleGodoosFile(filePath string) { - // 在这里添加对 .godoos 文件的具体处理逻辑 - log.Printf("Handling .godoos file: %s", filePath) +func handleGodoosFile(filePath string, knowledgeId uint) error { + log.Printf("========Handling .godoos file: %s", filePath) + baseName := filepath.Base(filePath) + if baseName[:8] != ".godoos." { + if baseName[:1] != "." { + office.ProcessFile(filePath, knowledgeId) + } + return nil + } + var doc office.Document + content, err := os.ReadFile(filePath) + if err != nil { + return err + } + err = json.Unmarshal(content, &doc) + if err != nil { + return err + } + if len(doc.Split) == 0 { + return fmt.Errorf("invalid .godoos file: %s", filePath) + } + knowData := GetVector(knowledgeId) + resList, err := server.GetEmbeddings(knowData.Engine, knowData.EmbeddingModel, doc.Split) + if err != nil { + return err + } + if len(resList) != len(doc.Split) { + return fmt.Errorf("invalid file len: %s, expected %d embeddings, got %d", filePath, len(doc.Split), len(resList)) + } + // var vectordocs []model.Vectordoc + // for i, res := range resList { + // //log.Printf("res: %v", res) + // vectordoc := model.Vectordoc{ + // Content: doc.Split[i], + // Embed: res, + // FilePath: filePath, + // KnowledgeID: knowledgeId, + // Pos: fmt.Sprintf("%d", i), + // } + // vectordocs = append(vectordocs, vectordoc) + // } + // result := vectorListDb.Create(&vectordocs) + // if result.Error != nil { + // return result.Error + // } + return nil } diff --git a/godo/ai/vector/model/vectorlist.go b/godo/ai/vector/model/vectorlist.go deleted file mode 100644 index 75a9f2f..0000000 --- a/godo/ai/vector/model/vectorlist.go +++ /dev/null @@ -1,13 +0,0 @@ -package model - -import "gorm.io/gorm" - -type VectorList struct { - gorm.Model - Name string `json:"name"` - FilePath string `json:"file_path"` - Engine string `json:"engine"` - EmbeddingUrl string `json:"embedding_url"` - EmbeddingApiKey string `json:"embedding_api_key"` - EmbeddingModel string `json:"embedding_model"` -} diff --git a/godo/ai/vector/vector.go b/godo/ai/vector/vector.go index c0529b5..6bc6aae 100644 --- a/godo/ai/vector/vector.go +++ b/godo/ai/vector/vector.go @@ -1,86 +1,301 @@ package vector import ( - "fmt" - "godo/ai/vector/model" + "encoding/json" "godo/libs" + "net/http" + "path/filepath" - _ "github.com/ncruces/go-sqlite3/embed" - "github.com/ncruces/go-sqlite3/gormlite" - "gorm.io/gorm" + _ "embed" + // sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/ncruces" + // "github.com/ncruces/go-sqlite3" ) -var vectorListDb *gorm.DB +//var VecDb *sqlx.DB -func init() { - dbPath := libs.GetVectorDb() - db, err := gorm.Open(gormlite.Open(dbPath), &gorm.Config{}) - if err != nil { - return - } - vectorListDb = db +type VectorList struct { + ID int `json:"id"` + FilePath string `json:"file_path"` + Engine string `json:"engine"` + EmbeddingModel string `json:"model"` +} +type VectorDoc struct { + ID int `json:"id"` + Content string `json:"content"` + FilePath string `json:"file_path"` + ListID int `json:"list_id"` +} +type VectorItem struct { + DocID int `json:"rowid"` + Embedding []float32 `json:"embedding"` } -// CreateVector 创建一个新的 VectorList 记录 -func CreateVector(name string, filePath string) (*gorm.DB, error) { - var list model.VectorList - if !libs.PathExists(filePath) { - return nil, fmt.Errorf("file path not exists") - } - // 检查是否已经存在同名的 VectorList - result := vectorListDb.Where("name = ?", name).First(&list) - if result.Error == nil { - return nil, fmt.Errorf("vector list with the same name already exists") - } +func init() { - // 创建新的 VectorList 记录 - newList := model.VectorList{ - Name: name, - FilePath: filePath, // 根据实际情况设置文件路径 - } + // dbPath := libs.GetVectorDb() + // sqlite_vec.Auto() - result = vectorListDb.Create(&newList) - if result.Error != nil { - return nil, result.Error - } + // db, err := sqlx.Connect("sqlite3", dbPath) + // if err != nil { + // fmt.Println("Failed to open SQLite database:", err) + // return + // } + // defer db.Close() + // VecDb = db + // dsn := "file:" + dbPath + // db, err := sqlite3.Open(dsn) + // //db, err := sqlite3.Open(":memory:") + // if err != nil { + // fmt.Println("Failed to open SQLite database:", err) + // return + // } + // stmt, _, err := db.Prepare(`SELECT vec_version()`) + // if err != nil { + // log.Fatal(err) + // } + + // stmt.Step() + // log.Printf("vec_version=%s\n", stmt.ColumnText(0)) + // stmt.Close() + // _, err = db.Exec("CREATE TABLE IF NOT EXISTS vec_list (id INTEGER PRIMARY KEY AUTOINCREMENT,file_path TEXT NOT NULL,engine TEXT NOT NULL,embedding_model TEXT NOT NULL)") + // if err != nil { + // log.Fatal(err) + // } + // _, err = db.Exec("CREATE TABLE IF NOT EXISTS vec_doc (id INTEGER PRIMARY KEY AUTOINCREMENT,list_id INTEGER DEFAULT 0,file_path TEXT,content TEXT)") + // if err != nil { + // log.Fatal(err) + // } + // _, err = db.Exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[768])") + // if err != nil { + // log.Fatal(err) + // } + // VecDb = db - return vectorListDb, nil + //InitMonitor() } -// DeleteVector 删除指定名称的 VectorList 记录 -func DeleteVector(name string) error { - result := vectorListDb.Where("name = ?", name).Delete(&model.VectorList{}) - if result.Error != nil { - return result.Error +func HandlerCreateKnowledge(w http.ResponseWriter, r *http.Request) { + var req VectorList + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + libs.ErrorMsg(w, "the chat request error:"+err.Error()) + return + } + if req.FilePath == "" { + libs.ErrorMsg(w, "file path is empty") + return } - if result.RowsAffected == 0 { - return fmt.Errorf("vector list not found") + basePath, err := libs.GetOsDir() + if err != nil { + libs.ErrorMsg(w, "get vector db path error:"+err.Error()) + return } + req.FilePath = filepath.Join(basePath, req.FilePath) - return nil + // id, err := CreateVector(req) + // if err != nil { + // libs.ErrorMsg(w, err.Error()) + // return + // } + // libs.SuccessMsg(w, id, "create vector success") } -// RenameVectorDb 更改指定名称的 VectorList 的数据库名称 -func RenameVectorDb(oldName string, newName string) error { - // 2. 获取旧的 VectorList 记录 - var oldList model.VectorList - result := vectorListDb.Where("name = ?", oldName).First(&oldList) - if result.Error != nil { - return fmt.Errorf("failed to find old vector list: %w", result.Error) - } +// // CreateVector 创建一个新的 VectorList 记录 +// func CreateVector(data VectorList) (uint, error) { +// if data.FilePath == "" { +// return 0, fmt.Errorf("file path is empty") +// } +// if data.Engine == "" { +// return 0, fmt.Errorf("engine is empty") +// } - // 5. 更新 VectorList 记录中的 DbPath 和 Name - oldList.Name = newName - result = vectorListDb.Save(&oldList) - if result.Error != nil { - return fmt.Errorf("failed to update vector list: %w", result.Error) - } +// if !libs.PathExists(data.FilePath) { +// return 0, fmt.Errorf("file path does not exist") +// } +// if data.EmbeddingModel == "" { +// return 0, fmt.Errorf("embedding model is empty") +// } - return nil -} +// // Check if a VectorList with the same path already exists + +// stmt, _, err := VecDb.Prepare(`SELECT id FROM vec_list WHERE file_path =`+ data.FilePath) +// if err != nil { +// log.Fatal(err) +// } +// defer stmt.Close() +// for stmt.Step() { +// fmt.Println(stmt.ColumnInt(0), stmt.ColumnText(1)) +// } +// if err := stmt.Err(); err != nil { +// log.Fatal(err) +// } + +// err = stmt.Close() +// if err != nil { +// log.Fatal(err) +// } +// // Create the new VectorList +// err = VecDb.Exec("INSERT INTO vec_list (file_path, engine, embedding_model) VALUES (?, ?, ?)", data.FilePath, data.Engine, data.EmbeddingModel) +// if err != nil { +// return 0, err +// } +// // Get the ID of the newly created VectorList +// vectorID, err := result.LastInsertId() +// if err != nil { +// return 0, err +// } + +// // Start background tasks +// go office.SetDocument(data.FilePath, uint(vectorID)) + +// return uint(vectorID), nil +// } + +// // DeleteVector 删除指定id的 VectorList 记录 +// func DeleteVector(id int) error { +// tx, err := VecDb.Begin() +// if err != nil { +// return err +// } +// defer tx.Rollback() + +// // Delete from vec_doc first +// _, err = tx.Exec("DELETE FROM vec_doc WHERE list_id = ?)", id) +// if err != nil { +// return err +// } + +// // Delete from vec_list +// result, err := tx.Exec("DELETE FROM vec_list WHERE id = ?", id) +// if err != nil { +// return err +// } + +// rowsAffected, err := result.RowsAffected() +// if err != nil { +// return err +// } +// if rowsAffected == 0 { +// return fmt.Errorf("vector list not found") +// } + +// return tx.Commit() +// } -func GetVectorList() []model.VectorList { - var vectorList []model.VectorList - vectorListDb.Find(&vectorList) +// // RenameVectorDb 更改指定名称的 VectorList 的数据库名称 +// func RenameVectorDb(oldName string, newName string) error { +// basePath, err := libs.GetOsDir() +// if err != nil { +// return fmt.Errorf("failed to find old vector list: %w", err) +// } + +// // 2. 获取旧的 VectorList 记录 +// var oldList VectorList +// oldPath := filepath.Join(basePath, oldName) +// err = VecDb.QueryRow("SELECT id FROM vec_list WHERE file_path = ?", oldPath).Scan(&oldList.ID) +// if err != nil { +// return fmt.Errorf("failed to find old vector list: %w", err) +// } +// MapFilePathMonitors[oldPath] = 0 + +// // 5. 更新 VectorList 记录中的 DbPath 和 Name +// newPath := filepath.Join(basePath, newName) +// _, err = VecDb.Exec("UPDATE vec_list SET file_path = ? WHERE id = ?", newPath, oldList.ID) +// if err != nil { +// return fmt.Errorf("failed to update vector list: %w", err) +// } +// MapFilePathMonitors[newPath] = oldList.ID + +// return nil +// } +// func InsertVectorDoc(data []VectorDoc, embedlist [][]float32) error { +// rowIds := map[int][]float32{} +// for i, v := range data { +// err := VecDb.Exec("INSERT INTO vec_doc (list_id, file_path, content) VALUES (?, ?, ?)", v.ListID, v.FilePath, v.Content) +// if err != nil { +// return err +// } +// rowID, err := result.LastInsertRowID() +// if err != nil { +// return err +// } +// rowid := int(rowID) +// rowIds[rowid] = embedlist[i] +// } +// stmt, err := VecDb.Prepare("INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)") +// if err != nil { +// log.Fatal(err) +// } +// defer stmt.Close() + +// for id, values := range rowIds { +// v, err := sqlite_vec.SerializeFloat32(values) +// if err != nil { +// log.Fatal(err) +// } +// err = stmt.BindInt64(1, int64(id)) +// if err != nil { +// log.Fatal(err) +// } +// err = stmt.BindBlob(2, v) +// if err != nil { +// log.Fatal(err) +// } +// err = stmt.Exec() +// if err != nil { +// log.Fatal(err) +// } +// stmt.Reset() +// } + +// return nil +// } +// func InitMonitor() { +// list := GetVectorList() +// for _, v := range list { +// MapFilePathMonitors[v.FilePath] = v.ID +// } +// FolderMonitor() +// } + +func GetVectorList() []VectorList { + var vectorList []VectorList + // stmt, _, err := VecDb.Prepare("SELECT id, file_path, engine, embedding_model FROM vec_list") + // if err != nil { + // fmt.Println("Failed to get vector list:", err) + // return vectorList + // } + // stmt.Step() + // log.Printf("vec_version=%s\n", stmt.ColumnText(0)) + // stmt.Close() + // defer rows.Close() + + // for rows.Next() { + // var v VectorList + // err := rows.Scan(&v.ID, &v.FilePath, &v.Engine, &v.EmbeddingModel) + // if err != nil { + // fmt.Println("Failed to scan vector list row:", err) + // continue + // } + // vectorList = append(vectorList, v) + // } + + return vectorList +} +func GetVector(id uint) VectorList { + var vectorList VectorList + // sql := "SELECT id, file_path, engine, embedding_model FROM vec_list WHERE id = " + fmt.Sprintf("%d", id) + // stmt, _, err := VecDb.Prepare(sql) + // if err != nil { + // fmt.Println("Failed to get vector list:", err) + // return vectorList + // } + // stmt.Step() + // log.Printf("vec_version=%s\n", stmt.ColumnText(0)) + // stmt.Close() + // err := VecDb.QueryRow("SELECT id, file_path, engine, embedding_model FROM vec_list WHERE id = ?", id).Scan(&vectorList.ID, &vectorList.FilePath, &vectorList.Engine, &vectorList.EmbeddingModel) + // if err != nil { + // fmt.Println("Failed to get vector:", err) + // } return vectorList } diff --git a/godo/cmd/main.go b/godo/cmd/main.go index 2257721..4122f3c 100644 --- a/godo/cmd/main.go +++ b/godo/cmd/main.go @@ -22,6 +22,7 @@ import ( "context" "godo/ai/search" model "godo/ai/server" + "godo/ai/vector" "godo/deps" "godo/files" "godo/libs" @@ -52,6 +53,7 @@ func OsStart() { db.InitDB() proxy.InitProxyHandlers() webdav.InitWebdav() + // vector.InitMonitor() router := mux.NewRouter() router.Use(recoverMiddleware) if libs.GetIsCors() { @@ -149,6 +151,7 @@ func OsStart() { aiRouter.HandleFunc("/chat", model.ChatHandler).Methods(http.MethodPost) aiRouter.HandleFunc("/embeddings", model.EmbeddingHandler).Methods(http.MethodPost) aiRouter.HandleFunc("/searchweb", search.SearchWebhandler).Methods(http.MethodGet) + aiRouter.HandleFunc("/addknowledge", vector.HandlerCreateKnowledge).Methods(http.MethodPost) //注册浏览器路由 ieRouter := router.PathPrefix("/ie").Subrouter() ieRouter.HandleFunc("/navigate", store.HandleNavigate).Methods(http.MethodGet) diff --git a/godo/go.mod b/godo/go.mod index c66af0d..944aa2d 100644 --- a/godo/go.mod +++ b/godo/go.mod @@ -6,29 +6,32 @@ toolchain go1.23.3 require ( github.com/EndFirstCorp/peekingReader v0.0.0-20171012052444-257fb6f1a1a6 + github.com/PuerkitoBio/goquery v1.10.1 + github.com/asg017/sqlite-vec-go-bindings v0.1.6 github.com/cavaliergopher/grab/v3 v3.0.1 github.com/ebitengine/purego v0.8.1 github.com/fsnotify/fsnotify v1.7.0 github.com/gorilla/mux v1.8.1 github.com/mattetti/filebuffer v1.0.1 github.com/minio/selfupdate v0.6.0 - github.com/ncruces/go-sqlite3 v0.21.1 + github.com/ncruces/go-sqlite3 v0.21.3 github.com/ncruces/go-sqlite3/gormlite v0.21.0 github.com/richardlehane/mscfb v1.0.4 github.com/shirou/gopsutil v3.21.11+incompatible golang.org/x/net v0.33.0 - golang.org/x/sys v0.28.0 + golang.org/x/sys v0.29.0 golang.org/x/text v0.21.0 gorm.io/gorm v1.25.12 + jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 ) require ( aead.dev/minisign v0.2.0 // indirect - github.com/PuerkitoBio/goquery v1.10.1 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/ncruces/julianday v1.0.0 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect @@ -41,5 +44,4 @@ require ( github.com/tklauser/numcpus v0.9.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/crypto v0.31.0 // indirect - jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 // indirect ) diff --git a/godo/go.sum b/godo/go.sum index 99bf89e..9b95803 100644 --- a/godo/go.sum +++ b/godo/go.sum @@ -1,11 +1,14 @@ aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/EndFirstCorp/peekingReader v0.0.0-20171012052444-257fb6f1a1a6 h1:t27CGFMv8DwGwqRPEa2VNof5I/aZwO6q2gfJhN8q0U4= github.com/EndFirstCorp/peekingReader v0.0.0-20171012052444-257fb6f1a1a6/go.mod h1:zpqkXxDsVfEIUZEWvT9yAo8OmRvSlRrcYQ3Zs8sSubA= github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU= github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/asg017/sqlite-vec-go-bindings v0.1.6 h1:Nx0jAzyS38XpkKznJ9xQjFXz2X9tI7KqjwVxV8RNoww= +github.com/asg017/sqlite-vec-go-bindings v0.1.6/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q= github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -17,6 +20,7 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= @@ -24,15 +28,19 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattetti/filebuffer v1.0.1 h1:gG7pyfnSIZCxdoKq+cPa8T0hhYtD9NxCdI4D7PTjRLM= github.com/mattetti/filebuffer v1.0.1/go.mod h1:YdMURNDOttIiruleeVr6f56OrMc+MydEnTcXwtkxNVs= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= -github.com/ncruces/go-sqlite3 v0.21.1 h1:cbzIOY3jQrXZWVsBfH9TCFj/iqqMIcJ7PLye4AAEwoQ= -github.com/ncruces/go-sqlite3 v0.21.1/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA= +github.com/ncruces/go-sqlite3 v0.21.3 h1:hHkfNQLcbnxPJZhC/RGw9SwP3bfkv/Y0xUHWsr1CdMQ= +github.com/ncruces/go-sqlite3 v0.21.3/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA= github.com/ncruces/go-sqlite3/gormlite v0.21.0 h1:9DsbvW9dS6uxXNFmbrNZixqAXKnIFnLM8oZmKqp8vcI= github.com/ncruces/go-sqlite3/gormlite v0.21.0/go.mod h1:rP4JXD6jlpOSsg2Ed++kzJIAZZCIBirVYqIpwaLW88E= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= @@ -87,8 +95,6 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -113,8 +119,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/godo/office/document.go b/godo/office/document.go index bf76fcb..d8eaa14 100644 --- a/godo/office/document.go +++ b/godo/office/document.go @@ -39,7 +39,7 @@ type DocResult struct { err error } -func SetDocument(dirPath string) error { +func SetDocument(dirPath string, knowledgeId uint) error { if !libs.PathExists(dirPath) { return nil } @@ -67,7 +67,7 @@ func SetDocument(dirPath string) error { wg.Add(1) go func(filePath string) { defer wg.Done() - result := ProcessFile(filePath) + result := ProcessFile(filePath, knowledgeId) results <- result }(filePath) } @@ -94,7 +94,7 @@ func SetDocument(dirPath string) error { return nil } -func ProcessFile(filePath string) DocResult { +func ProcessFile(filePath string, knowledgeId uint) DocResult { doc, err := GetDocument(filePath) if err != nil { return DocResult{filePath: filePath, err: err} @@ -105,9 +105,11 @@ func ProcessFile(filePath string) DocResult { return DocResult{filePath: filePath, err: err} } - newFileName := ".godoos." + filepath.Base(filePath) + //newFileName := ".godoos." + filepath.Base(filePath) + newFileName := fmt.Sprintf(".godoos.%d.%s.json", knowledgeId, filepath.Base(filePath)) newFilePath := filepath.Join(filepath.Dir(filePath), newFileName) - + // log.Printf("New file name: %s", newFileName) + // log.Printf("New file path: %s", newFilePath) err = os.WriteFile(newFilePath, jsonData, 0644) if err != nil { return DocResult{filePath: filePath, err: err} diff --git a/godo/vector/go.mod b/godo/vector/go.mod new file mode 100644 index 0000000..452c4f0 --- /dev/null +++ b/godo/vector/go.mod @@ -0,0 +1,14 @@ +module vector + +go 1.22.5 + +require ( + github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37 + github.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7 +) + +require ( + github.com/ncruces/julianday v1.0.0 // indirect + github.com/tetratelabs/wazero v1.7.3 // indirect + golang.org/x/sys v0.22.0 // indirect +) diff --git a/godo/vector/go.sum b/godo/vector/go.sum new file mode 100644 index 0000000..3dcd8e9 --- /dev/null +++ b/godo/vector/go.sum @@ -0,0 +1,12 @@ +github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37 h1:Gz6YkDCs60k5VwbBPKDfAPPeIBcuaN3qriAozAaIIZI= +github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q= +github.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7 h1:ssM02uUFDfz0V2TMg2du2BjbW9cpOhFJK0kpDN+X768= +github.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7/go.mod h1:FnCyui8SlDoL0mQZ5dTouNo7s7jXS0kJv9lBt1GlM9w= +github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= +github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= +github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw= +github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= diff --git a/godo/vector/init.go b/godo/vector/init.go new file mode 100644 index 0000000..4b3245b --- /dev/null +++ b/godo/vector/init.go @@ -0,0 +1,5 @@ +package main + +func Init() { + +} diff --git a/godo/vector/main.go b/godo/vector/main.go new file mode 100644 index 0000000..ffd7078 --- /dev/null +++ b/godo/vector/main.go @@ -0,0 +1,100 @@ +package main + +import ( + _ "embed" + "log" + + sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/ncruces" + "github.com/ncruces/go-sqlite3" +) + +var Db *sqlite3.Conn + +func main() { + db, err := sqlite3.Open(":memory:") + if err != nil { + log.Fatal(err) + } + + stmt, _, err := db.Prepare(`SELECT sqlite_version(), vec_version()`) + if err != nil { + log.Fatal(err) + } + + stmt.Step() + + log.Printf("sqlite_version=%s, vec_version=%s\n", stmt.ColumnText(0), stmt.ColumnText(1)) + stmt.Close() + + err = db.Exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])") + if err != nil { + log.Fatal(err) + } + items := map[int][]float32{ + 1: {0.1, 0.1, 0.1, 0.1}, + 2: {0.2, 0.2, 0.2, 0.2}, + 3: {0.3, 0.3, 0.3, 0.3}, + 4: {0.4, 0.4, 0.4, 0.4}, + 5: {0.5, 0.5, 0.5, 0.5}, + } + q := []float32{0.3, 0.3, 0.3, 0.3} + + stmt, _, err = db.Prepare("INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)") + if err != nil { + log.Fatal(err) + } + + for id, values := range items { + v, err := sqlite_vec.SerializeFloat32(values) + if err != nil { + log.Fatal(err) + } + stmt.BindInt(1, id) + stmt.BindBlob(2, v) + err = stmt.Exec() + if err != nil { + log.Fatal(err) + } + stmt.Reset() + } + stmt.Close() + + stmt, _, err = db.Prepare(` + SELECT + rowid, + distance + FROM vec_items + WHERE embedding MATCH ? + ORDER BY distance + LIMIT 3 + `) + + if err != nil { + log.Fatal(err) + } + + query, err := sqlite_vec.SerializeFloat32(q) + if err != nil { + log.Fatal(err) + } + stmt.BindBlob(1, query) + + for stmt.Step() { + rowid := stmt.ColumnInt64(0) + distance := stmt.ColumnFloat(1) + log.Printf("rowid=%d, distance=%f\n", rowid, distance) + } + if err := stmt.Err(); err != nil { + log.Fatal(err) + } + + err = stmt.Close() + if err != nil { + log.Fatal(err) + } + + err = db.Close() + if err != nil { + log.Fatal(err) + } +} diff --git a/test/go.mod b/test/go.mod new file mode 100644 index 0000000..0dbedbb --- /dev/null +++ b/test/go.mod @@ -0,0 +1,7 @@ +module godoos + +go 1.23.2 + +require github.com/fsnotify/fsnotify v1.8.0 + +require golang.org/x/sys v0.13.0 // indirect diff --git a/test/go.sum b/test/go.sum new file mode 100644 index 0000000..b6b937f --- /dev/null +++ b/test/go.sum @@ -0,0 +1,4 @@ +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/test/main.go b/test/main.go new file mode 100644 index 0000000..88e2d5b --- /dev/null +++ b/test/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "log" + + "github.com/fsnotify/fsnotify" +) + +func main() { + // Create new watcher. + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + // Start listening for events. + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + log.Println("event:", event) + if event.Has(fsnotify.Write) { + log.Println("modified file:", event.Name) + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + log.Println("error:", err) + } + } + }() + + // Add a path. + err = watcher.Add("E:\\test3\\E\\新建文件夹") + if err != nil { + log.Fatal(err) + } + + // Block main goroutine forever. + <-make(chan struct{}) +}