Browse Source

change embed

master
godo 5 months ago
parent
commit
9db2b235c9
  1. 3
      README.md
  2. 21
      frontend/src/components/builtin/FileList.vue
  3. 25
      frontend/src/hook/useAi.ts
  4. 75
      godo/ai/server/embedding.go
  5. 177
      godo/ai/vector/document.go
  6. 13
      godo/ai/vector/model/vectorlist.go
  7. 337
      godo/ai/vector/vector.go
  8. 3
      godo/cmd/main.go
  9. 10
      godo/go.mod
  10. 17
      godo/go.sum
  11. 12
      godo/office/document.go
  12. 14
      godo/vector/go.mod
  13. 12
      godo/vector/go.sum
  14. 5
      godo/vector/init.go
  15. 100
      godo/vector/main.go
  16. 7
      test/go.mod
  17. 4
      test/go.sum
  18. 46
      test/main.go

3
README.md

@ -17,8 +17,9 @@
- 新增配置本地代理和远程代理,本地代理可实现本机ip映射外部域名,远程代理内嵌frpc设置后可实现内网用户外网域名访问。
- 修改锁屏机制,确保外网访问安全。
- 支持本地聊天ai对话文件和联网搜索。
## 🏭 第三阶段目标(十二月底发布)
## 🏭 第三阶段目标(月底发布)
1. **文档处理与Markdown智能升级**:(已完成)
- **AI续写**:借助先进的自然语言处理技术,让您的文档创作灵感不断,续写流畅无阻。
- **智能总结**:一键提取文档精髓,快速生成精炼总结,助力高效阅读与信息提炼。

21
frontend/src/components/builtin/FileList.vue

@ -72,6 +72,7 @@ import { dealSystemName, t } from "@/i18n";
import { useChooseStore } from "@/stores/choose";
import { getSystemKey } from "@/system/config";
import { emitEvent, mountEvent } from "@/system/event";
import { addKnowledge } from "@/hook/useAi";
import {
basename,
BrowserWindow,
@ -343,6 +344,26 @@ function handleRightClick(
},
});
}
menuArr.push({
label: "加入知识库",
click: () => {
//console.log(item.path)
addKnowledge(item.path).then((res:any) => {
console.log(res)
if(res.code != 0){
new Notify({
title: t("tips"),
content: res.message,
});
}else{
new Notify({
title: t("tips"),
content: "添加成功",
});
}
})
},
});
}
if (choose.ifShow) {
menuArr.push({

25
frontend/src/hook/useAi.ts

@ -61,4 +61,29 @@ export async function askAi(question: any, action: string) {
}
}
export async function addKnowledge(path: string) {
const modelStore = useModelStore();
const config = getSystemConfig();
const model = await modelStore.getModel('embeddings')
if (!model) {
return {
code:-1,
message:'请先设置嵌入模型'
}
}
const apiUrl = config.aiUrl + '/ai/addknowledge'
const postMsg: any = {
engine: model.info.engine,
model: model.model,
file_path: path
};
const complain = await fetchPost(apiUrl, JSON.stringify(postMsg))
if (!complain.ok) {
return {
code:-1,
message:'请求失败'
}
}
return await complain.json()
}

75
godo/ai/server/embedding.go

@ -0,0 +1,75 @@
package server
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
)
func GetEmbeddings(engine string, model string, text []string) ([][]float32, error) {
if engine == "ollama" {
res, err := getOllamaEmbedding(model, text)
if err != nil {
log.Printf("getOllamaEmbedding error: %v", err)
return nil, err
}
return res, nil
}
return nil, nil
}
func getOllamaEmbedding(model string, text []string) ([][]float32, error) {
url := GetOllamaUrl() + "/api/embed"
//log.Printf("url: %s", url)
client := &http.Client{}
// Prepare the request body.
reqBody, err := json.Marshal(map[string]interface{}{
"model": model,
"input": text,
})
//log.Printf("reqBody: %s", reqBody)
if err != nil {
return nil, fmt.Errorf("couldn't marshal request body: %w", err)
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
if err != nil {
return nil, fmt.Errorf("couldn't create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
// Send the request.
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("couldn't send request: %w", err)
}
defer resp.Body.Close()
// Check the response status.
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("error response from the embedding API: %s", resp.Status)
}
// Read and decode the response body.
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("couldn't read response body: %w", err)
}
var embeddingResponse struct {
Embeddings [][]float32 `json:"embeddings"`
}
err = json.Unmarshal(body, &embeddingResponse)
if err != nil {
return nil, fmt.Errorf("couldn't unmarshal response body: %w", err)
}
log.Printf("Embedding: %v", embeddingResponse.Embeddings)
// Return the embeddings directly.
if len(embeddingResponse.Embeddings) == 0 {
return nil, fmt.Errorf("no embeddings returned from the API")
}
return embeddingResponse.Embeddings, nil
}

177
godo/ai/vector/document.go

@ -1,87 +1,178 @@
package vector
import (
"encoding/json"
"fmt"
"godo/ai/server"
"godo/libs"
"godo/office"
"log"
"path"
"sync"
"os"
"path/filepath"
"strings"
"github.com/fsnotify/fsnotify"
)
const numWorkers = 5 // 设置 worker 数量
var MapFilePathMonitors = map[string]uint{}
func MonitorFolder(folderPath string) {
if !libs.PathExists(folderPath) {
func FolderMonitor() {
basePath, err := libs.GetOsDir()
if err != nil {
log.Printf("Error getting base path: %s", err.Error())
return
}
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatalf("Failed to create watcher for folder %s: %v", folderPath, err)
log.Printf("Error creating watcher: %s", err.Error())
return
}
defer watcher.Close()
fileQueue := make(chan string, 100) // 创建文件路径队列
var wg sync.WaitGroup
// 启动 worker goroutine
for i := 0; i < numWorkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for filePath := range fileQueue {
handleGodoosFile(filePath)
}
}()
}
// 递归添加所有子目录
addRecursive(basePath, watcher)
done := make(chan bool)
// Start listening for events.
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
log.Println("error:", err)
return
}
log.Println("event:", event)
if event.Op&(fsnotify.Create|fsnotify.Write) != 0 { // 监听创建和修改事件
baseName := path.Base(event.Name)
if baseName[:8] == ".godoos." { // 检查文件名是否以 .godoos 开头
log.Printf("Detected .godoos file: %s", event.Name)
fileQueue <- event.Name // 将文件路径放入队列
} else {
if baseName[:1] == "." {
return
//log.Println("event:", event)
filePath := event.Name
result, knowledgeId := shouldProcess(filePath)
//log.Printf("result:%d,knowledgeId:%d", result, knowledgeId)
if result > 0 {
info, err := os.Stat(filePath)
if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) {
log.Println("modified file:", filePath)
if !info.IsDir() {
handleGodoosFile(filePath, knowledgeId)
}
}
if event.Has(fsnotify.Create) || event.Has(fsnotify.Rename) {
// 处理创建或重命名事件,添加新目录
if err == nil && info.IsDir() {
addRecursive(filePath, watcher)
}
}
if event.Has(fsnotify.Remove) {
// 处理删除事件,移除目录
if err == nil && info.IsDir() {
watcher.Remove(filePath)
}
office.ProcessFile(event.Name)
}
}
case err, ok := <-watcher.Errors:
if !ok {
return
}
log.Printf("Error watching folder %s: %v", folderPath, err)
log.Println("error:", err)
}
}
}()
err = watcher.Add(folderPath)
// Add a path.
err = watcher.Add(basePath)
if err != nil {
log.Fatalf("Failed to add folder %s to watcher: %v", folderPath, err)
log.Fatal(err)
}
// 关闭文件队列
go func() {
<-done
close(fileQueue)
wg.Wait()
}()
// Block main goroutine forever.
<-make(chan struct{})
}
func shouldProcess(filePath string) (int, uint) {
// 规范化路径
filePath = filepath.Clean(filePath)
// 检查文件路径是否在 MapFilePathMonitors 中
for path, id := range MapFilePathMonitors {
if id < 1 {
return 0, 0
}
path = filepath.Clean(path)
if filePath == path {
return 1, id // 完全相等
}
if strings.HasPrefix(filePath, path+string(filepath.Separator)) {
return 2, id // 包含
}
}
return 0, 0 // 不存在
}
func addRecursive(path string, watcher *fsnotify.Watcher) {
err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
if err != nil {
log.Printf("Error walking path %s: %v", path, err)
return err
}
if info.IsDir() {
result, _ := shouldProcess(path)
if result > 0 {
if err := watcher.Add(path); err != nil {
log.Printf("Error adding path %s to watcher: %v", path, err)
return err
}
log.Printf("Added path %s to watcher", path)
}
<-done
}
return nil
})
if err != nil {
log.Printf("Error adding recursive paths: %v", err)
}
}
func handleGodoosFile(filePath string) {
// 在这里添加对 .godoos 文件的具体处理逻辑
log.Printf("Handling .godoos file: %s", filePath)
func handleGodoosFile(filePath string, knowledgeId uint) error {
log.Printf("========Handling .godoos file: %s", filePath)
baseName := filepath.Base(filePath)
if baseName[:8] != ".godoos." {
if baseName[:1] != "." {
office.ProcessFile(filePath, knowledgeId)
}
return nil
}
var doc office.Document
content, err := os.ReadFile(filePath)
if err != nil {
return err
}
err = json.Unmarshal(content, &doc)
if err != nil {
return err
}
if len(doc.Split) == 0 {
return fmt.Errorf("invalid .godoos file: %s", filePath)
}
knowData := GetVector(knowledgeId)
resList, err := server.GetEmbeddings(knowData.Engine, knowData.EmbeddingModel, doc.Split)
if err != nil {
return err
}
if len(resList) != len(doc.Split) {
return fmt.Errorf("invalid file len: %s, expected %d embeddings, got %d", filePath, len(doc.Split), len(resList))
}
// var vectordocs []model.Vectordoc
// for i, res := range resList {
// //log.Printf("res: %v", res)
// vectordoc := model.Vectordoc{
// Content: doc.Split[i],
// Embed: res,
// FilePath: filePath,
// KnowledgeID: knowledgeId,
// Pos: fmt.Sprintf("%d", i),
// }
// vectordocs = append(vectordocs, vectordoc)
// }
// result := vectorListDb.Create(&vectordocs)
// if result.Error != nil {
// return result.Error
// }
return nil
}

13
godo/ai/vector/model/vectorlist.go

@ -1,13 +0,0 @@
package model
import "gorm.io/gorm"
type VectorList struct {
gorm.Model
Name string `json:"name"`
FilePath string `json:"file_path"`
Engine string `json:"engine"`
EmbeddingUrl string `json:"embedding_url"`
EmbeddingApiKey string `json:"embedding_api_key"`
EmbeddingModel string `json:"embedding_model"`
}

337
godo/ai/vector/vector.go

@ -1,86 +1,301 @@
package vector
import (
"fmt"
"godo/ai/vector/model"
"encoding/json"
"godo/libs"
"net/http"
"path/filepath"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/gormlite"
"gorm.io/gorm"
_ "embed"
// sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/ncruces"
// "github.com/ncruces/go-sqlite3"
)
var vectorListDb *gorm.DB
//var VecDb *sqlx.DB
func init() {
dbPath := libs.GetVectorDb()
db, err := gorm.Open(gormlite.Open(dbPath), &gorm.Config{})
if err != nil {
return
}
vectorListDb = db
type VectorList struct {
ID int `json:"id"`
FilePath string `json:"file_path"`
Engine string `json:"engine"`
EmbeddingModel string `json:"model"`
}
type VectorDoc struct {
ID int `json:"id"`
Content string `json:"content"`
FilePath string `json:"file_path"`
ListID int `json:"list_id"`
}
type VectorItem struct {
DocID int `json:"rowid"`
Embedding []float32 `json:"embedding"`
}
// CreateVector 创建一个新的 VectorList 记录
func CreateVector(name string, filePath string) (*gorm.DB, error) {
var list model.VectorList
if !libs.PathExists(filePath) {
return nil, fmt.Errorf("file path not exists")
}
// 检查是否已经存在同名的 VectorList
result := vectorListDb.Where("name = ?", name).First(&list)
if result.Error == nil {
return nil, fmt.Errorf("vector list with the same name already exists")
}
func init() {
// 创建新的 VectorList 记录
newList := model.VectorList{
Name: name,
FilePath: filePath, // 根据实际情况设置文件路径
}
// dbPath := libs.GetVectorDb()
// sqlite_vec.Auto()
result = vectorListDb.Create(&newList)
if result.Error != nil {
return nil, result.Error
}
// db, err := sqlx.Connect("sqlite3", dbPath)
// if err != nil {
// fmt.Println("Failed to open SQLite database:", err)
// return
// }
// defer db.Close()
// VecDb = db
// dsn := "file:" + dbPath
// db, err := sqlite3.Open(dsn)
// //db, err := sqlite3.Open(":memory:")
// if err != nil {
// fmt.Println("Failed to open SQLite database:", err)
// return
// }
// stmt, _, err := db.Prepare(`SELECT vec_version()`)
// if err != nil {
// log.Fatal(err)
// }
// stmt.Step()
// log.Printf("vec_version=%s\n", stmt.ColumnText(0))
// stmt.Close()
// _, err = db.Exec("CREATE TABLE IF NOT EXISTS vec_list (id INTEGER PRIMARY KEY AUTOINCREMENT,file_path TEXT NOT NULL,engine TEXT NOT NULL,embedding_model TEXT NOT NULL)")
// if err != nil {
// log.Fatal(err)
// }
// _, err = db.Exec("CREATE TABLE IF NOT EXISTS vec_doc (id INTEGER PRIMARY KEY AUTOINCREMENT,list_id INTEGER DEFAULT 0,file_path TEXT,content TEXT)")
// if err != nil {
// log.Fatal(err)
// }
// _, err = db.Exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[768])")
// if err != nil {
// log.Fatal(err)
// }
// VecDb = db
return vectorListDb, nil
//InitMonitor()
}
// DeleteVector 删除指定名称的 VectorList 记录
func DeleteVector(name string) error {
result := vectorListDb.Where("name = ?", name).Delete(&model.VectorList{})
if result.Error != nil {
return result.Error
func HandlerCreateKnowledge(w http.ResponseWriter, r *http.Request) {
var req VectorList
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
libs.ErrorMsg(w, "the chat request error:"+err.Error())
return
}
if req.FilePath == "" {
libs.ErrorMsg(w, "file path is empty")
return
}
if result.RowsAffected == 0 {
return fmt.Errorf("vector list not found")
basePath, err := libs.GetOsDir()
if err != nil {
libs.ErrorMsg(w, "get vector db path error:"+err.Error())
return
}
req.FilePath = filepath.Join(basePath, req.FilePath)
return nil
// id, err := CreateVector(req)
// if err != nil {
// libs.ErrorMsg(w, err.Error())
// return
// }
// libs.SuccessMsg(w, id, "create vector success")
}
// RenameVectorDb 更改指定名称的 VectorList 的数据库名称
func RenameVectorDb(oldName string, newName string) error {
// 2. 获取旧的 VectorList 记录
var oldList model.VectorList
result := vectorListDb.Where("name = ?", oldName).First(&oldList)
if result.Error != nil {
return fmt.Errorf("failed to find old vector list: %w", result.Error)
}
// // CreateVector 创建一个新的 VectorList 记录
// func CreateVector(data VectorList) (uint, error) {
// if data.FilePath == "" {
// return 0, fmt.Errorf("file path is empty")
// }
// if data.Engine == "" {
// return 0, fmt.Errorf("engine is empty")
// }
// 5. 更新 VectorList 记录中的 DbPath 和 Name
oldList.Name = newName
result = vectorListDb.Save(&oldList)
if result.Error != nil {
return fmt.Errorf("failed to update vector list: %w", result.Error)
}
// if !libs.PathExists(data.FilePath) {
// return 0, fmt.Errorf("file path does not exist")
// }
// if data.EmbeddingModel == "" {
// return 0, fmt.Errorf("embedding model is empty")
// }
return nil
}
// // Check if a VectorList with the same path already exists
// stmt, _, err := VecDb.Prepare(`SELECT id FROM vec_list WHERE file_path =`+ data.FilePath)
// if err != nil {
// log.Fatal(err)
// }
// defer stmt.Close()
// for stmt.Step() {
// fmt.Println(stmt.ColumnInt(0), stmt.ColumnText(1))
// }
// if err := stmt.Err(); err != nil {
// log.Fatal(err)
// }
// err = stmt.Close()
// if err != nil {
// log.Fatal(err)
// }
// // Create the new VectorList
// err = VecDb.Exec("INSERT INTO vec_list (file_path, engine, embedding_model) VALUES (?, ?, ?)", data.FilePath, data.Engine, data.EmbeddingModel)
// if err != nil {
// return 0, err
// }
// // Get the ID of the newly created VectorList
// vectorID, err := result.LastInsertId()
// if err != nil {
// return 0, err
// }
// // Start background tasks
// go office.SetDocument(data.FilePath, uint(vectorID))
// return uint(vectorID), nil
// }
// // DeleteVector 删除指定id的 VectorList 记录
// func DeleteVector(id int) error {
// tx, err := VecDb.Begin()
// if err != nil {
// return err
// }
// defer tx.Rollback()
// // Delete from vec_doc first
// _, err = tx.Exec("DELETE FROM vec_doc WHERE list_id = ?)", id)
// if err != nil {
// return err
// }
// // Delete from vec_list
// result, err := tx.Exec("DELETE FROM vec_list WHERE id = ?", id)
// if err != nil {
// return err
// }
// rowsAffected, err := result.RowsAffected()
// if err != nil {
// return err
// }
// if rowsAffected == 0 {
// return fmt.Errorf("vector list not found")
// }
// return tx.Commit()
// }
func GetVectorList() []model.VectorList {
var vectorList []model.VectorList
vectorListDb.Find(&vectorList)
// // RenameVectorDb 更改指定名称的 VectorList 的数据库名称
// func RenameVectorDb(oldName string, newName string) error {
// basePath, err := libs.GetOsDir()
// if err != nil {
// return fmt.Errorf("failed to find old vector list: %w", err)
// }
// // 2. 获取旧的 VectorList 记录
// var oldList VectorList
// oldPath := filepath.Join(basePath, oldName)
// err = VecDb.QueryRow("SELECT id FROM vec_list WHERE file_path = ?", oldPath).Scan(&oldList.ID)
// if err != nil {
// return fmt.Errorf("failed to find old vector list: %w", err)
// }
// MapFilePathMonitors[oldPath] = 0
// // 5. 更新 VectorList 记录中的 DbPath 和 Name
// newPath := filepath.Join(basePath, newName)
// _, err = VecDb.Exec("UPDATE vec_list SET file_path = ? WHERE id = ?", newPath, oldList.ID)
// if err != nil {
// return fmt.Errorf("failed to update vector list: %w", err)
// }
// MapFilePathMonitors[newPath] = oldList.ID
// return nil
// }
// func InsertVectorDoc(data []VectorDoc, embedlist [][]float32) error {
// rowIds := map[int][]float32{}
// for i, v := range data {
// err := VecDb.Exec("INSERT INTO vec_doc (list_id, file_path, content) VALUES (?, ?, ?)", v.ListID, v.FilePath, v.Content)
// if err != nil {
// return err
// }
// rowID, err := result.LastInsertRowID()
// if err != nil {
// return err
// }
// rowid := int(rowID)
// rowIds[rowid] = embedlist[i]
// }
// stmt, err := VecDb.Prepare("INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)")
// if err != nil {
// log.Fatal(err)
// }
// defer stmt.Close()
// for id, values := range rowIds {
// v, err := sqlite_vec.SerializeFloat32(values)
// if err != nil {
// log.Fatal(err)
// }
// err = stmt.BindInt64(1, int64(id))
// if err != nil {
// log.Fatal(err)
// }
// err = stmt.BindBlob(2, v)
// if err != nil {
// log.Fatal(err)
// }
// err = stmt.Exec()
// if err != nil {
// log.Fatal(err)
// }
// stmt.Reset()
// }
// return nil
// }
// func InitMonitor() {
// list := GetVectorList()
// for _, v := range list {
// MapFilePathMonitors[v.FilePath] = v.ID
// }
// FolderMonitor()
// }
func GetVectorList() []VectorList {
var vectorList []VectorList
// stmt, _, err := VecDb.Prepare("SELECT id, file_path, engine, embedding_model FROM vec_list")
// if err != nil {
// fmt.Println("Failed to get vector list:", err)
// return vectorList
// }
// stmt.Step()
// log.Printf("vec_version=%s\n", stmt.ColumnText(0))
// stmt.Close()
// defer rows.Close()
// for rows.Next() {
// var v VectorList
// err := rows.Scan(&v.ID, &v.FilePath, &v.Engine, &v.EmbeddingModel)
// if err != nil {
// fmt.Println("Failed to scan vector list row:", err)
// continue
// }
// vectorList = append(vectorList, v)
// }
return vectorList
}
func GetVector(id uint) VectorList {
var vectorList VectorList
// sql := "SELECT id, file_path, engine, embedding_model FROM vec_list WHERE id = " + fmt.Sprintf("%d", id)
// stmt, _, err := VecDb.Prepare(sql)
// if err != nil {
// fmt.Println("Failed to get vector list:", err)
// return vectorList
// }
// stmt.Step()
// log.Printf("vec_version=%s\n", stmt.ColumnText(0))
// stmt.Close()
// err := VecDb.QueryRow("SELECT id, file_path, engine, embedding_model FROM vec_list WHERE id = ?", id).Scan(&vectorList.ID, &vectorList.FilePath, &vectorList.Engine, &vectorList.EmbeddingModel)
// if err != nil {
// fmt.Println("Failed to get vector:", err)
// }
return vectorList
}

3
godo/cmd/main.go

@ -22,6 +22,7 @@ import (
"context"
"godo/ai/search"
model "godo/ai/server"
"godo/ai/vector"
"godo/deps"
"godo/files"
"godo/libs"
@ -52,6 +53,7 @@ func OsStart() {
db.InitDB()
proxy.InitProxyHandlers()
webdav.InitWebdav()
// vector.InitMonitor()
router := mux.NewRouter()
router.Use(recoverMiddleware)
if libs.GetIsCors() {
@ -149,6 +151,7 @@ func OsStart() {
aiRouter.HandleFunc("/chat", model.ChatHandler).Methods(http.MethodPost)
aiRouter.HandleFunc("/embeddings", model.EmbeddingHandler).Methods(http.MethodPost)
aiRouter.HandleFunc("/searchweb", search.SearchWebhandler).Methods(http.MethodGet)
aiRouter.HandleFunc("/addknowledge", vector.HandlerCreateKnowledge).Methods(http.MethodPost)
//注册浏览器路由
ieRouter := router.PathPrefix("/ie").Subrouter()
ieRouter.HandleFunc("/navigate", store.HandleNavigate).Methods(http.MethodGet)

10
godo/go.mod

@ -6,29 +6,32 @@ toolchain go1.23.3
require (
github.com/EndFirstCorp/peekingReader v0.0.0-20171012052444-257fb6f1a1a6
github.com/PuerkitoBio/goquery v1.10.1
github.com/asg017/sqlite-vec-go-bindings v0.1.6
github.com/cavaliergopher/grab/v3 v3.0.1
github.com/ebitengine/purego v0.8.1
github.com/fsnotify/fsnotify v1.7.0
github.com/gorilla/mux v1.8.1
github.com/mattetti/filebuffer v1.0.1
github.com/minio/selfupdate v0.6.0
github.com/ncruces/go-sqlite3 v0.21.1
github.com/ncruces/go-sqlite3 v0.21.3
github.com/ncruces/go-sqlite3/gormlite v0.21.0
github.com/richardlehane/mscfb v1.0.4
github.com/shirou/gopsutil v3.21.11+incompatible
golang.org/x/net v0.33.0
golang.org/x/sys v0.28.0
golang.org/x/sys v0.29.0
golang.org/x/text v0.21.0
gorm.io/gorm v1.25.12
jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056
)
require (
aead.dev/minisign v0.2.0 // indirect
github.com/PuerkitoBio/goquery v1.10.1 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmoiron/sqlx v1.4.0 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
@ -41,5 +44,4 @@ require (
github.com/tklauser/numcpus v0.9.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
golang.org/x/crypto v0.31.0 // indirect
jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 // indirect
)

17
godo/go.sum

@ -1,11 +1,14 @@
aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk=
aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/EndFirstCorp/peekingReader v0.0.0-20171012052444-257fb6f1a1a6 h1:t27CGFMv8DwGwqRPEa2VNof5I/aZwO6q2gfJhN8q0U4=
github.com/EndFirstCorp/peekingReader v0.0.0-20171012052444-257fb6f1a1a6/go.mod h1:zpqkXxDsVfEIUZEWvT9yAo8OmRvSlRrcYQ3Zs8sSubA=
github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU=
github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/asg017/sqlite-vec-go-bindings v0.1.6 h1:Nx0jAzyS38XpkKznJ9xQjFXz2X9tI7KqjwVxV8RNoww=
github.com/asg017/sqlite-vec-go-bindings v0.1.6/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q=
github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4=
github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@ -17,6 +20,7 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
@ -24,15 +28,19 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattetti/filebuffer v1.0.1 h1:gG7pyfnSIZCxdoKq+cPa8T0hhYtD9NxCdI4D7PTjRLM=
github.com/mattetti/filebuffer v1.0.1/go.mod h1:YdMURNDOttIiruleeVr6f56OrMc+MydEnTcXwtkxNVs=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
github.com/ncruces/go-sqlite3 v0.21.1 h1:cbzIOY3jQrXZWVsBfH9TCFj/iqqMIcJ7PLye4AAEwoQ=
github.com/ncruces/go-sqlite3 v0.21.1/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA=
github.com/ncruces/go-sqlite3 v0.21.3 h1:hHkfNQLcbnxPJZhC/RGw9SwP3bfkv/Y0xUHWsr1CdMQ=
github.com/ncruces/go-sqlite3 v0.21.3/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA=
github.com/ncruces/go-sqlite3/gormlite v0.21.0 h1:9DsbvW9dS6uxXNFmbrNZixqAXKnIFnLM8oZmKqp8vcI=
github.com/ncruces/go-sqlite3/gormlite v0.21.0/go.mod h1:rP4JXD6jlpOSsg2Ed++kzJIAZZCIBirVYqIpwaLW88E=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
@ -87,8 +95,6 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -113,8 +119,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

12
godo/office/document.go

@ -39,7 +39,7 @@ type DocResult struct {
err error
}
func SetDocument(dirPath string) error {
func SetDocument(dirPath string, knowledgeId uint) error {
if !libs.PathExists(dirPath) {
return nil
}
@ -67,7 +67,7 @@ func SetDocument(dirPath string) error {
wg.Add(1)
go func(filePath string) {
defer wg.Done()
result := ProcessFile(filePath)
result := ProcessFile(filePath, knowledgeId)
results <- result
}(filePath)
}
@ -94,7 +94,7 @@ func SetDocument(dirPath string) error {
return nil
}
func ProcessFile(filePath string) DocResult {
func ProcessFile(filePath string, knowledgeId uint) DocResult {
doc, err := GetDocument(filePath)
if err != nil {
return DocResult{filePath: filePath, err: err}
@ -105,9 +105,11 @@ func ProcessFile(filePath string) DocResult {
return DocResult{filePath: filePath, err: err}
}
newFileName := ".godoos." + filepath.Base(filePath)
//newFileName := ".godoos." + filepath.Base(filePath)
newFileName := fmt.Sprintf(".godoos.%d.%s.json", knowledgeId, filepath.Base(filePath))
newFilePath := filepath.Join(filepath.Dir(filePath), newFileName)
// log.Printf("New file name: %s", newFileName)
// log.Printf("New file path: %s", newFilePath)
err = os.WriteFile(newFilePath, jsonData, 0644)
if err != nil {
return DocResult{filePath: filePath, err: err}

14
godo/vector/go.mod

@ -0,0 +1,14 @@
module vector
go 1.22.5
require (
github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37
github.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7
)
require (
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.7.3 // indirect
golang.org/x/sys v0.22.0 // indirect
)

12
godo/vector/go.sum

@ -0,0 +1,12 @@
github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37 h1:Gz6YkDCs60k5VwbBPKDfAPPeIBcuaN3qriAozAaIIZI=
github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q=
github.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7 h1:ssM02uUFDfz0V2TMg2du2BjbW9cpOhFJK0kpDN+X768=
github.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7/go.mod h1:FnCyui8SlDoL0mQZ5dTouNo7s7jXS0kJv9lBt1GlM9w=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw=
github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=

5
godo/vector/init.go

@ -0,0 +1,5 @@
package main
func Init() {
}

100
godo/vector/main.go

@ -0,0 +1,100 @@
package main
import (
_ "embed"
"log"
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/ncruces"
"github.com/ncruces/go-sqlite3"
)
var Db *sqlite3.Conn
func main() {
db, err := sqlite3.Open(":memory:")
if err != nil {
log.Fatal(err)
}
stmt, _, err := db.Prepare(`SELECT sqlite_version(), vec_version()`)
if err != nil {
log.Fatal(err)
}
stmt.Step()
log.Printf("sqlite_version=%s, vec_version=%s\n", stmt.ColumnText(0), stmt.ColumnText(1))
stmt.Close()
err = db.Exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])")
if err != nil {
log.Fatal(err)
}
items := map[int][]float32{
1: {0.1, 0.1, 0.1, 0.1},
2: {0.2, 0.2, 0.2, 0.2},
3: {0.3, 0.3, 0.3, 0.3},
4: {0.4, 0.4, 0.4, 0.4},
5: {0.5, 0.5, 0.5, 0.5},
}
q := []float32{0.3, 0.3, 0.3, 0.3}
stmt, _, err = db.Prepare("INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)")
if err != nil {
log.Fatal(err)
}
for id, values := range items {
v, err := sqlite_vec.SerializeFloat32(values)
if err != nil {
log.Fatal(err)
}
stmt.BindInt(1, id)
stmt.BindBlob(2, v)
err = stmt.Exec()
if err != nil {
log.Fatal(err)
}
stmt.Reset()
}
stmt.Close()
stmt, _, err = db.Prepare(`
SELECT
rowid,
distance
FROM vec_items
WHERE embedding MATCH ?
ORDER BY distance
LIMIT 3
`)
if err != nil {
log.Fatal(err)
}
query, err := sqlite_vec.SerializeFloat32(q)
if err != nil {
log.Fatal(err)
}
stmt.BindBlob(1, query)
for stmt.Step() {
rowid := stmt.ColumnInt64(0)
distance := stmt.ColumnFloat(1)
log.Printf("rowid=%d, distance=%f\n", rowid, distance)
}
if err := stmt.Err(); err != nil {
log.Fatal(err)
}
err = stmt.Close()
if err != nil {
log.Fatal(err)
}
err = db.Close()
if err != nil {
log.Fatal(err)
}
}

7
test/go.mod

@ -0,0 +1,7 @@
module godoos
go 1.23.2
require github.com/fsnotify/fsnotify v1.8.0
require golang.org/x/sys v0.13.0 // indirect

4
test/go.sum

@ -0,0 +1,4 @@
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

46
test/main.go

@ -0,0 +1,46 @@
package main
import (
"log"
"github.com/fsnotify/fsnotify"
)
func main() {
// Create new watcher.
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
defer watcher.Close()
// Start listening for events.
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return
}
log.Println("event:", event)
if event.Has(fsnotify.Write) {
log.Println("modified file:", event.Name)
}
case err, ok := <-watcher.Errors:
if !ok {
return
}
log.Println("error:", err)
}
}
}()
// Add a path.
err = watcher.Add("E:\\test3\\E\\新建文件夹")
if err != nil {
log.Fatal(err)
}
// Block main goroutine forever.
<-make(chan struct{})
}
Loading…
Cancel
Save