mirror of https://gitee.com/godoos/godoos.git
18 changed files with 749 additions and 132 deletions
@ -0,0 +1,75 @@ |
|||
package server |
|||
|
|||
import ( |
|||
"bytes" |
|||
"encoding/json" |
|||
"fmt" |
|||
"io" |
|||
"log" |
|||
"net/http" |
|||
) |
|||
|
|||
func GetEmbeddings(engine string, model string, text []string) ([][]float32, error) { |
|||
if engine == "ollama" { |
|||
res, err := getOllamaEmbedding(model, text) |
|||
if err != nil { |
|||
log.Printf("getOllamaEmbedding error: %v", err) |
|||
return nil, err |
|||
} |
|||
return res, nil |
|||
} |
|||
return nil, nil |
|||
} |
|||
|
|||
func getOllamaEmbedding(model string, text []string) ([][]float32, error) { |
|||
url := GetOllamaUrl() + "/api/embed" |
|||
//log.Printf("url: %s", url)
|
|||
client := &http.Client{} |
|||
|
|||
// Prepare the request body.
|
|||
reqBody, err := json.Marshal(map[string]interface{}{ |
|||
"model": model, |
|||
"input": text, |
|||
}) |
|||
//log.Printf("reqBody: %s", reqBody)
|
|||
if err != nil { |
|||
return nil, fmt.Errorf("couldn't marshal request body: %w", err) |
|||
} |
|||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody)) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("couldn't create request: %w", err) |
|||
} |
|||
req.Header.Set("Content-Type", "application/json") |
|||
|
|||
// Send the request.
|
|||
resp, err := client.Do(req) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("couldn't send request: %w", err) |
|||
} |
|||
defer resp.Body.Close() |
|||
|
|||
// Check the response status.
|
|||
if resp.StatusCode != http.StatusOK { |
|||
return nil, fmt.Errorf("error response from the embedding API: %s", resp.Status) |
|||
} |
|||
|
|||
// Read and decode the response body.
|
|||
body, err := io.ReadAll(resp.Body) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("couldn't read response body: %w", err) |
|||
} |
|||
var embeddingResponse struct { |
|||
Embeddings [][]float32 `json:"embeddings"` |
|||
} |
|||
err = json.Unmarshal(body, &embeddingResponse) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("couldn't unmarshal response body: %w", err) |
|||
} |
|||
log.Printf("Embedding: %v", embeddingResponse.Embeddings) |
|||
|
|||
// Return the embeddings directly.
|
|||
if len(embeddingResponse.Embeddings) == 0 { |
|||
return nil, fmt.Errorf("no embeddings returned from the API") |
|||
} |
|||
return embeddingResponse.Embeddings, nil |
|||
} |
@ -1,87 +1,178 @@ |
|||
package vector |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"fmt" |
|||
"godo/ai/server" |
|||
"godo/libs" |
|||
"godo/office" |
|||
"log" |
|||
"path" |
|||
"sync" |
|||
"os" |
|||
"path/filepath" |
|||
"strings" |
|||
|
|||
"github.com/fsnotify/fsnotify" |
|||
) |
|||
|
|||
const numWorkers = 5 // 设置 worker 数量
|
|||
var MapFilePathMonitors = map[string]uint{} |
|||
|
|||
func MonitorFolder(folderPath string) { |
|||
if !libs.PathExists(folderPath) { |
|||
func FolderMonitor() { |
|||
basePath, err := libs.GetOsDir() |
|||
if err != nil { |
|||
log.Printf("Error getting base path: %s", err.Error()) |
|||
return |
|||
} |
|||
watcher, err := fsnotify.NewWatcher() |
|||
if err != nil { |
|||
log.Fatalf("Failed to create watcher for folder %s: %v", folderPath, err) |
|||
log.Printf("Error creating watcher: %s", err.Error()) |
|||
return |
|||
} |
|||
defer watcher.Close() |
|||
|
|||
fileQueue := make(chan string, 100) // 创建文件路径队列
|
|||
var wg sync.WaitGroup |
|||
|
|||
// 启动 worker goroutine
|
|||
for i := 0; i < numWorkers; i++ { |
|||
wg.Add(1) |
|||
go func() { |
|||
defer wg.Done() |
|||
for filePath := range fileQueue { |
|||
handleGodoosFile(filePath) |
|||
} |
|||
}() |
|||
} |
|||
// 递归添加所有子目录
|
|||
addRecursive(basePath, watcher) |
|||
|
|||
done := make(chan bool) |
|||
// Start listening for events.
|
|||
go func() { |
|||
for { |
|||
select { |
|||
case event, ok := <-watcher.Events: |
|||
if !ok { |
|||
log.Println("error:", err) |
|||
return |
|||
} |
|||
log.Println("event:", event) |
|||
if event.Op&(fsnotify.Create|fsnotify.Write) != 0 { // 监听创建和修改事件
|
|||
baseName := path.Base(event.Name) |
|||
if baseName[:8] == ".godoos." { // 检查文件名是否以 .godoos 开头
|
|||
log.Printf("Detected .godoos file: %s", event.Name) |
|||
fileQueue <- event.Name // 将文件路径放入队列
|
|||
} else { |
|||
if baseName[:1] == "." { |
|||
return |
|||
//log.Println("event:", event)
|
|||
filePath := event.Name |
|||
result, knowledgeId := shouldProcess(filePath) |
|||
//log.Printf("result:%d,knowledgeId:%d", result, knowledgeId)
|
|||
if result > 0 { |
|||
info, err := os.Stat(filePath) |
|||
if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) { |
|||
log.Println("modified file:", filePath) |
|||
if !info.IsDir() { |
|||
handleGodoosFile(filePath, knowledgeId) |
|||
} |
|||
} |
|||
if event.Has(fsnotify.Create) || event.Has(fsnotify.Rename) { |
|||
// 处理创建或重命名事件,添加新目录
|
|||
if err == nil && info.IsDir() { |
|||
addRecursive(filePath, watcher) |
|||
} |
|||
} |
|||
if event.Has(fsnotify.Remove) { |
|||
// 处理删除事件,移除目录
|
|||
if err == nil && info.IsDir() { |
|||
watcher.Remove(filePath) |
|||
} |
|||
office.ProcessFile(event.Name) |
|||
} |
|||
} |
|||
case err, ok := <-watcher.Errors: |
|||
if !ok { |
|||
return |
|||
} |
|||
log.Printf("Error watching folder %s: %v", folderPath, err) |
|||
log.Println("error:", err) |
|||
} |
|||
} |
|||
}() |
|||
|
|||
err = watcher.Add(folderPath) |
|||
// Add a path.
|
|||
err = watcher.Add(basePath) |
|||
if err != nil { |
|||
log.Fatalf("Failed to add folder %s to watcher: %v", folderPath, err) |
|||
log.Fatal(err) |
|||
} |
|||
|
|||
// 关闭文件队列
|
|||
go func() { |
|||
<-done |
|||
close(fileQueue) |
|||
wg.Wait() |
|||
}() |
|||
// Block main goroutine forever.
|
|||
<-make(chan struct{}) |
|||
} |
|||
|
|||
func shouldProcess(filePath string) (int, uint) { |
|||
// 规范化路径
|
|||
filePath = filepath.Clean(filePath) |
|||
|
|||
// 检查文件路径是否在 MapFilePathMonitors 中
|
|||
for path, id := range MapFilePathMonitors { |
|||
if id < 1 { |
|||
return 0, 0 |
|||
} |
|||
path = filepath.Clean(path) |
|||
if filePath == path { |
|||
return 1, id // 完全相等
|
|||
} |
|||
if strings.HasPrefix(filePath, path+string(filepath.Separator)) { |
|||
return 2, id // 包含
|
|||
} |
|||
} |
|||
return 0, 0 // 不存在
|
|||
} |
|||
|
|||
func addRecursive(path string, watcher *fsnotify.Watcher) { |
|||
err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { |
|||
if err != nil { |
|||
log.Printf("Error walking path %s: %v", path, err) |
|||
return err |
|||
} |
|||
if info.IsDir() { |
|||
result, _ := shouldProcess(path) |
|||
if result > 0 { |
|||
if err := watcher.Add(path); err != nil { |
|||
log.Printf("Error adding path %s to watcher: %v", path, err) |
|||
return err |
|||
} |
|||
log.Printf("Added path %s to watcher", path) |
|||
} |
|||
|
|||
<-done |
|||
} |
|||
return nil |
|||
}) |
|||
if err != nil { |
|||
log.Printf("Error adding recursive paths: %v", err) |
|||
} |
|||
} |
|||
|
|||
func handleGodoosFile(filePath string) { |
|||
// 在这里添加对 .godoos 文件的具体处理逻辑
|
|||
log.Printf("Handling .godoos file: %s", filePath) |
|||
func handleGodoosFile(filePath string, knowledgeId uint) error { |
|||
log.Printf("========Handling .godoos file: %s", filePath) |
|||
baseName := filepath.Base(filePath) |
|||
if baseName[:8] != ".godoos." { |
|||
if baseName[:1] != "." { |
|||
office.ProcessFile(filePath, knowledgeId) |
|||
} |
|||
return nil |
|||
} |
|||
var doc office.Document |
|||
content, err := os.ReadFile(filePath) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
err = json.Unmarshal(content, &doc) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if len(doc.Split) == 0 { |
|||
return fmt.Errorf("invalid .godoos file: %s", filePath) |
|||
} |
|||
knowData := GetVector(knowledgeId) |
|||
resList, err := server.GetEmbeddings(knowData.Engine, knowData.EmbeddingModel, doc.Split) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if len(resList) != len(doc.Split) { |
|||
return fmt.Errorf("invalid file len: %s, expected %d embeddings, got %d", filePath, len(doc.Split), len(resList)) |
|||
} |
|||
// var vectordocs []model.Vectordoc
|
|||
// for i, res := range resList {
|
|||
// //log.Printf("res: %v", res)
|
|||
// vectordoc := model.Vectordoc{
|
|||
// Content: doc.Split[i],
|
|||
// Embed: res,
|
|||
// FilePath: filePath,
|
|||
// KnowledgeID: knowledgeId,
|
|||
// Pos: fmt.Sprintf("%d", i),
|
|||
// }
|
|||
// vectordocs = append(vectordocs, vectordoc)
|
|||
// }
|
|||
// result := vectorListDb.Create(&vectordocs)
|
|||
// if result.Error != nil {
|
|||
// return result.Error
|
|||
// }
|
|||
return nil |
|||
} |
|||
|
@ -1,13 +0,0 @@ |
|||
package model |
|||
|
|||
import "gorm.io/gorm" |
|||
|
|||
type VectorList struct { |
|||
gorm.Model |
|||
Name string `json:"name"` |
|||
FilePath string `json:"file_path"` |
|||
Engine string `json:"engine"` |
|||
EmbeddingUrl string `json:"embedding_url"` |
|||
EmbeddingApiKey string `json:"embedding_api_key"` |
|||
EmbeddingModel string `json:"embedding_model"` |
|||
} |
@ -1,86 +1,301 @@ |
|||
package vector |
|||
|
|||
import ( |
|||
"fmt" |
|||
"godo/ai/vector/model" |
|||
"encoding/json" |
|||
"godo/libs" |
|||
"net/http" |
|||
"path/filepath" |
|||
|
|||
_ "github.com/ncruces/go-sqlite3/embed" |
|||
"github.com/ncruces/go-sqlite3/gormlite" |
|||
"gorm.io/gorm" |
|||
_ "embed" |
|||
// sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/ncruces"
|
|||
// "github.com/ncruces/go-sqlite3"
|
|||
) |
|||
|
|||
var vectorListDb *gorm.DB |
|||
//var VecDb *sqlx.DB
|
|||
|
|||
func init() { |
|||
dbPath := libs.GetVectorDb() |
|||
db, err := gorm.Open(gormlite.Open(dbPath), &gorm.Config{}) |
|||
if err != nil { |
|||
return |
|||
} |
|||
vectorListDb = db |
|||
type VectorList struct { |
|||
ID int `json:"id"` |
|||
FilePath string `json:"file_path"` |
|||
Engine string `json:"engine"` |
|||
EmbeddingModel string `json:"model"` |
|||
} |
|||
type VectorDoc struct { |
|||
ID int `json:"id"` |
|||
Content string `json:"content"` |
|||
FilePath string `json:"file_path"` |
|||
ListID int `json:"list_id"` |
|||
} |
|||
type VectorItem struct { |
|||
DocID int `json:"rowid"` |
|||
Embedding []float32 `json:"embedding"` |
|||
} |
|||
|
|||
// CreateVector 创建一个新的 VectorList 记录
|
|||
func CreateVector(name string, filePath string) (*gorm.DB, error) { |
|||
var list model.VectorList |
|||
if !libs.PathExists(filePath) { |
|||
return nil, fmt.Errorf("file path not exists") |
|||
} |
|||
// 检查是否已经存在同名的 VectorList
|
|||
result := vectorListDb.Where("name = ?", name).First(&list) |
|||
if result.Error == nil { |
|||
return nil, fmt.Errorf("vector list with the same name already exists") |
|||
} |
|||
func init() { |
|||
|
|||
// 创建新的 VectorList 记录
|
|||
newList := model.VectorList{ |
|||
Name: name, |
|||
FilePath: filePath, // 根据实际情况设置文件路径
|
|||
} |
|||
// dbPath := libs.GetVectorDb()
|
|||
// sqlite_vec.Auto()
|
|||
|
|||
result = vectorListDb.Create(&newList) |
|||
if result.Error != nil { |
|||
return nil, result.Error |
|||
} |
|||
// db, err := sqlx.Connect("sqlite3", dbPath)
|
|||
// if err != nil {
|
|||
// fmt.Println("Failed to open SQLite database:", err)
|
|||
// return
|
|||
// }
|
|||
// defer db.Close()
|
|||
// VecDb = db
|
|||
// dsn := "file:" + dbPath
|
|||
// db, err := sqlite3.Open(dsn)
|
|||
// //db, err := sqlite3.Open(":memory:")
|
|||
// if err != nil {
|
|||
// fmt.Println("Failed to open SQLite database:", err)
|
|||
// return
|
|||
// }
|
|||
// stmt, _, err := db.Prepare(`SELECT vec_version()`)
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
|
|||
// stmt.Step()
|
|||
// log.Printf("vec_version=%s\n", stmt.ColumnText(0))
|
|||
// stmt.Close()
|
|||
// _, err = db.Exec("CREATE TABLE IF NOT EXISTS vec_list (id INTEGER PRIMARY KEY AUTOINCREMENT,file_path TEXT NOT NULL,engine TEXT NOT NULL,embedding_model TEXT NOT NULL)")
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// _, err = db.Exec("CREATE TABLE IF NOT EXISTS vec_doc (id INTEGER PRIMARY KEY AUTOINCREMENT,list_id INTEGER DEFAULT 0,file_path TEXT,content TEXT)")
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// _, err = db.Exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[768])")
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// VecDb = db
|
|||
|
|||
return vectorListDb, nil |
|||
//InitMonitor()
|
|||
} |
|||
|
|||
// DeleteVector 删除指定名称的 VectorList 记录
|
|||
func DeleteVector(name string) error { |
|||
result := vectorListDb.Where("name = ?", name).Delete(&model.VectorList{}) |
|||
if result.Error != nil { |
|||
return result.Error |
|||
func HandlerCreateKnowledge(w http.ResponseWriter, r *http.Request) { |
|||
var req VectorList |
|||
err := json.NewDecoder(r.Body).Decode(&req) |
|||
if err != nil { |
|||
libs.ErrorMsg(w, "the chat request error:"+err.Error()) |
|||
return |
|||
} |
|||
if req.FilePath == "" { |
|||
libs.ErrorMsg(w, "file path is empty") |
|||
return |
|||
} |
|||
if result.RowsAffected == 0 { |
|||
return fmt.Errorf("vector list not found") |
|||
basePath, err := libs.GetOsDir() |
|||
if err != nil { |
|||
libs.ErrorMsg(w, "get vector db path error:"+err.Error()) |
|||
return |
|||
} |
|||
req.FilePath = filepath.Join(basePath, req.FilePath) |
|||
|
|||
return nil |
|||
// id, err := CreateVector(req)
|
|||
// if err != nil {
|
|||
// libs.ErrorMsg(w, err.Error())
|
|||
// return
|
|||
// }
|
|||
// libs.SuccessMsg(w, id, "create vector success")
|
|||
} |
|||
|
|||
// RenameVectorDb 更改指定名称的 VectorList 的数据库名称
|
|||
func RenameVectorDb(oldName string, newName string) error { |
|||
// 2. 获取旧的 VectorList 记录
|
|||
var oldList model.VectorList |
|||
result := vectorListDb.Where("name = ?", oldName).First(&oldList) |
|||
if result.Error != nil { |
|||
return fmt.Errorf("failed to find old vector list: %w", result.Error) |
|||
} |
|||
// // CreateVector 创建一个新的 VectorList 记录
|
|||
// func CreateVector(data VectorList) (uint, error) {
|
|||
// if data.FilePath == "" {
|
|||
// return 0, fmt.Errorf("file path is empty")
|
|||
// }
|
|||
// if data.Engine == "" {
|
|||
// return 0, fmt.Errorf("engine is empty")
|
|||
// }
|
|||
|
|||
// 5. 更新 VectorList 记录中的 DbPath 和 Name
|
|||
oldList.Name = newName |
|||
result = vectorListDb.Save(&oldList) |
|||
if result.Error != nil { |
|||
return fmt.Errorf("failed to update vector list: %w", result.Error) |
|||
} |
|||
// if !libs.PathExists(data.FilePath) {
|
|||
// return 0, fmt.Errorf("file path does not exist")
|
|||
// }
|
|||
// if data.EmbeddingModel == "" {
|
|||
// return 0, fmt.Errorf("embedding model is empty")
|
|||
// }
|
|||
|
|||
return nil |
|||
} |
|||
// // Check if a VectorList with the same path already exists
|
|||
|
|||
// stmt, _, err := VecDb.Prepare(`SELECT id FROM vec_list WHERE file_path =`+ data.FilePath)
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// defer stmt.Close()
|
|||
// for stmt.Step() {
|
|||
// fmt.Println(stmt.ColumnInt(0), stmt.ColumnText(1))
|
|||
// }
|
|||
// if err := stmt.Err(); err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
|
|||
// err = stmt.Close()
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// // Create the new VectorList
|
|||
// err = VecDb.Exec("INSERT INTO vec_list (file_path, engine, embedding_model) VALUES (?, ?, ?)", data.FilePath, data.Engine, data.EmbeddingModel)
|
|||
// if err != nil {
|
|||
// return 0, err
|
|||
// }
|
|||
// // Get the ID of the newly created VectorList
|
|||
// vectorID, err := result.LastInsertId()
|
|||
// if err != nil {
|
|||
// return 0, err
|
|||
// }
|
|||
|
|||
// // Start background tasks
|
|||
// go office.SetDocument(data.FilePath, uint(vectorID))
|
|||
|
|||
// return uint(vectorID), nil
|
|||
// }
|
|||
|
|||
// // DeleteVector 删除指定id的 VectorList 记录
|
|||
// func DeleteVector(id int) error {
|
|||
// tx, err := VecDb.Begin()
|
|||
// if err != nil {
|
|||
// return err
|
|||
// }
|
|||
// defer tx.Rollback()
|
|||
|
|||
// // Delete from vec_doc first
|
|||
// _, err = tx.Exec("DELETE FROM vec_doc WHERE list_id = ?)", id)
|
|||
// if err != nil {
|
|||
// return err
|
|||
// }
|
|||
|
|||
// // Delete from vec_list
|
|||
// result, err := tx.Exec("DELETE FROM vec_list WHERE id = ?", id)
|
|||
// if err != nil {
|
|||
// return err
|
|||
// }
|
|||
|
|||
// rowsAffected, err := result.RowsAffected()
|
|||
// if err != nil {
|
|||
// return err
|
|||
// }
|
|||
// if rowsAffected == 0 {
|
|||
// return fmt.Errorf("vector list not found")
|
|||
// }
|
|||
|
|||
// return tx.Commit()
|
|||
// }
|
|||
|
|||
func GetVectorList() []model.VectorList { |
|||
var vectorList []model.VectorList |
|||
vectorListDb.Find(&vectorList) |
|||
// // RenameVectorDb 更改指定名称的 VectorList 的数据库名称
|
|||
// func RenameVectorDb(oldName string, newName string) error {
|
|||
// basePath, err := libs.GetOsDir()
|
|||
// if err != nil {
|
|||
// return fmt.Errorf("failed to find old vector list: %w", err)
|
|||
// }
|
|||
|
|||
// // 2. 获取旧的 VectorList 记录
|
|||
// var oldList VectorList
|
|||
// oldPath := filepath.Join(basePath, oldName)
|
|||
// err = VecDb.QueryRow("SELECT id FROM vec_list WHERE file_path = ?", oldPath).Scan(&oldList.ID)
|
|||
// if err != nil {
|
|||
// return fmt.Errorf("failed to find old vector list: %w", err)
|
|||
// }
|
|||
// MapFilePathMonitors[oldPath] = 0
|
|||
|
|||
// // 5. 更新 VectorList 记录中的 DbPath 和 Name
|
|||
// newPath := filepath.Join(basePath, newName)
|
|||
// _, err = VecDb.Exec("UPDATE vec_list SET file_path = ? WHERE id = ?", newPath, oldList.ID)
|
|||
// if err != nil {
|
|||
// return fmt.Errorf("failed to update vector list: %w", err)
|
|||
// }
|
|||
// MapFilePathMonitors[newPath] = oldList.ID
|
|||
|
|||
// return nil
|
|||
// }
|
|||
// func InsertVectorDoc(data []VectorDoc, embedlist [][]float32) error {
|
|||
// rowIds := map[int][]float32{}
|
|||
// for i, v := range data {
|
|||
// err := VecDb.Exec("INSERT INTO vec_doc (list_id, file_path, content) VALUES (?, ?, ?)", v.ListID, v.FilePath, v.Content)
|
|||
// if err != nil {
|
|||
// return err
|
|||
// }
|
|||
// rowID, err := result.LastInsertRowID()
|
|||
// if err != nil {
|
|||
// return err
|
|||
// }
|
|||
// rowid := int(rowID)
|
|||
// rowIds[rowid] = embedlist[i]
|
|||
// }
|
|||
// stmt, err := VecDb.Prepare("INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)")
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// defer stmt.Close()
|
|||
|
|||
// for id, values := range rowIds {
|
|||
// v, err := sqlite_vec.SerializeFloat32(values)
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// err = stmt.BindInt64(1, int64(id))
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// err = stmt.BindBlob(2, v)
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// err = stmt.Exec()
|
|||
// if err != nil {
|
|||
// log.Fatal(err)
|
|||
// }
|
|||
// stmt.Reset()
|
|||
// }
|
|||
|
|||
// return nil
|
|||
// }
|
|||
// func InitMonitor() {
|
|||
// list := GetVectorList()
|
|||
// for _, v := range list {
|
|||
// MapFilePathMonitors[v.FilePath] = v.ID
|
|||
// }
|
|||
// FolderMonitor()
|
|||
// }
|
|||
|
|||
func GetVectorList() []VectorList { |
|||
var vectorList []VectorList |
|||
// stmt, _, err := VecDb.Prepare("SELECT id, file_path, engine, embedding_model FROM vec_list")
|
|||
// if err != nil {
|
|||
// fmt.Println("Failed to get vector list:", err)
|
|||
// return vectorList
|
|||
// }
|
|||
// stmt.Step()
|
|||
// log.Printf("vec_version=%s\n", stmt.ColumnText(0))
|
|||
// stmt.Close()
|
|||
// defer rows.Close()
|
|||
|
|||
// for rows.Next() {
|
|||
// var v VectorList
|
|||
// err := rows.Scan(&v.ID, &v.FilePath, &v.Engine, &v.EmbeddingModel)
|
|||
// if err != nil {
|
|||
// fmt.Println("Failed to scan vector list row:", err)
|
|||
// continue
|
|||
// }
|
|||
// vectorList = append(vectorList, v)
|
|||
// }
|
|||
|
|||
return vectorList |
|||
} |
|||
func GetVector(id uint) VectorList { |
|||
var vectorList VectorList |
|||
// sql := "SELECT id, file_path, engine, embedding_model FROM vec_list WHERE id = " + fmt.Sprintf("%d", id)
|
|||
// stmt, _, err := VecDb.Prepare(sql)
|
|||
// if err != nil {
|
|||
// fmt.Println("Failed to get vector list:", err)
|
|||
// return vectorList
|
|||
// }
|
|||
// stmt.Step()
|
|||
// log.Printf("vec_version=%s\n", stmt.ColumnText(0))
|
|||
// stmt.Close()
|
|||
// err := VecDb.QueryRow("SELECT id, file_path, engine, embedding_model FROM vec_list WHERE id = ?", id).Scan(&vectorList.ID, &vectorList.FilePath, &vectorList.Engine, &vectorList.EmbeddingModel)
|
|||
// if err != nil {
|
|||
// fmt.Println("Failed to get vector:", err)
|
|||
// }
|
|||
return vectorList |
|||
} |
|||
|
@ -0,0 +1,14 @@ |
|||
module vector |
|||
|
|||
go 1.22.5 |
|||
|
|||
require ( |
|||
github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37 |
|||
github.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7 |
|||
) |
|||
|
|||
require ( |
|||
github.com/ncruces/julianday v1.0.0 // indirect |
|||
github.com/tetratelabs/wazero v1.7.3 // indirect |
|||
golang.org/x/sys v0.22.0 // indirect |
|||
) |
@ -0,0 +1,12 @@ |
|||
github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37 h1:Gz6YkDCs60k5VwbBPKDfAPPeIBcuaN3qriAozAaIIZI= |
|||
github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q= |
|||
github.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7 h1:ssM02uUFDfz0V2TMg2du2BjbW9cpOhFJK0kpDN+X768= |
|||
github.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7/go.mod h1:FnCyui8SlDoL0mQZ5dTouNo7s7jXS0kJv9lBt1GlM9w= |
|||
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= |
|||
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= |
|||
github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw= |
|||
github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= |
|||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= |
|||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= |
|||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= |
|||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= |
@ -0,0 +1,5 @@ |
|||
package main |
|||
|
|||
func Init() { |
|||
|
|||
} |
@ -0,0 +1,100 @@ |
|||
package main |
|||
|
|||
import ( |
|||
_ "embed" |
|||
"log" |
|||
|
|||
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/ncruces" |
|||
"github.com/ncruces/go-sqlite3" |
|||
) |
|||
|
|||
var Db *sqlite3.Conn |
|||
|
|||
func main() { |
|||
db, err := sqlite3.Open(":memory:") |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
|
|||
stmt, _, err := db.Prepare(`SELECT sqlite_version(), vec_version()`) |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
|
|||
stmt.Step() |
|||
|
|||
log.Printf("sqlite_version=%s, vec_version=%s\n", stmt.ColumnText(0), stmt.ColumnText(1)) |
|||
stmt.Close() |
|||
|
|||
err = db.Exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])") |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
items := map[int][]float32{ |
|||
1: {0.1, 0.1, 0.1, 0.1}, |
|||
2: {0.2, 0.2, 0.2, 0.2}, |
|||
3: {0.3, 0.3, 0.3, 0.3}, |
|||
4: {0.4, 0.4, 0.4, 0.4}, |
|||
5: {0.5, 0.5, 0.5, 0.5}, |
|||
} |
|||
q := []float32{0.3, 0.3, 0.3, 0.3} |
|||
|
|||
stmt, _, err = db.Prepare("INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)") |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
|
|||
for id, values := range items { |
|||
v, err := sqlite_vec.SerializeFloat32(values) |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
stmt.BindInt(1, id) |
|||
stmt.BindBlob(2, v) |
|||
err = stmt.Exec() |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
stmt.Reset() |
|||
} |
|||
stmt.Close() |
|||
|
|||
stmt, _, err = db.Prepare(` |
|||
SELECT |
|||
rowid, |
|||
distance |
|||
FROM vec_items |
|||
WHERE embedding MATCH ? |
|||
ORDER BY distance |
|||
LIMIT 3 |
|||
`) |
|||
|
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
|
|||
query, err := sqlite_vec.SerializeFloat32(q) |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
stmt.BindBlob(1, query) |
|||
|
|||
for stmt.Step() { |
|||
rowid := stmt.ColumnInt64(0) |
|||
distance := stmt.ColumnFloat(1) |
|||
log.Printf("rowid=%d, distance=%f\n", rowid, distance) |
|||
} |
|||
if err := stmt.Err(); err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
|
|||
err = stmt.Close() |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
|
|||
err = db.Close() |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
} |
@ -0,0 +1,7 @@ |
|||
module godoos |
|||
|
|||
go 1.23.2 |
|||
|
|||
require github.com/fsnotify/fsnotify v1.8.0 |
|||
|
|||
require golang.org/x/sys v0.13.0 // indirect |
@ -0,0 +1,4 @@ |
|||
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= |
|||
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= |
|||
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= |
|||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
@ -0,0 +1,46 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"log" |
|||
|
|||
"github.com/fsnotify/fsnotify" |
|||
) |
|||
|
|||
func main() { |
|||
// Create new watcher.
|
|||
watcher, err := fsnotify.NewWatcher() |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
defer watcher.Close() |
|||
|
|||
// Start listening for events.
|
|||
go func() { |
|||
for { |
|||
select { |
|||
case event, ok := <-watcher.Events: |
|||
if !ok { |
|||
return |
|||
} |
|||
log.Println("event:", event) |
|||
if event.Has(fsnotify.Write) { |
|||
log.Println("modified file:", event.Name) |
|||
} |
|||
case err, ok := <-watcher.Errors: |
|||
if !ok { |
|||
return |
|||
} |
|||
log.Println("error:", err) |
|||
} |
|||
} |
|||
}() |
|||
|
|||
// Add a path.
|
|||
err = watcher.Add("E:\\test3\\E\\新建文件夹") |
|||
if err != nil { |
|||
log.Fatal(err) |
|||
} |
|||
|
|||
// Block main goroutine forever.
|
|||
<-make(chan struct{}) |
|||
} |
Loading…
Reference in new issue