Browse Source

change ai dirs

master
godo 6 months ago
parent
commit
decf66a7a1
  1. 0
      godo/ai/api/gitee.go
  2. 0
      godo/ai/api/openai.go
  3. 47
      godo/ai/config/data.go
  4. 45
      godo/ai/config/libs.go
  5. 2
      godo/ai/convert/doc.go
  6. 0
      godo/ai/convert/doc/clx.go
  7. 0
      godo/ai/convert/doc/doc.go
  8. 0
      godo/ai/convert/doc/fib.go
  9. 0
      godo/ai/convert/doc/plcFld.go
  10. 0
      godo/ai/convert/doc/utf16.go
  11. 0
      godo/ai/convert/docx.go
  12. 2
      godo/ai/convert/epub.go
  13. 0
      godo/ai/convert/find.go
  14. 0
      godo/ai/convert/html.go
  15. 0
      godo/ai/convert/http.go
  16. 2
      godo/ai/convert/image.go
  17. 2
      godo/ai/convert/img.go
  18. 0
      godo/ai/convert/libs/file.go
  19. 0
      godo/ai/convert/libs/kind.go
  20. 0
      godo/ai/convert/libs/rapid.go
  21. 0
      godo/ai/convert/libs/xpdf.go
  22. 0
      godo/ai/convert/main.go
  23. 0
      godo/ai/convert/main_test.go
  24. 0
      godo/ai/convert/md.go
  25. 0
      godo/ai/convert/odt.go
  26. 2
      godo/ai/convert/pdf.go
  27. 0
      godo/ai/convert/pptx.go
  28. 0
      godo/ai/convert/rtf.go
  29. 0
      godo/ai/convert/tidy.go
  30. 0
      godo/ai/convert/txt.go
  31. 0
      godo/ai/convert/url.go
  32. 2
      godo/ai/convert/xlsx.go
  33. 0
      godo/ai/convert/xml.go
  34. 2
      godo/ai/server/chat.go
  35. 2
      godo/ai/server/common.go
  36. 24
      godo/ai/server/down.go
  37. 135
      godo/ai/server/ollama.go
  38. 20
      godo/ai/server/server.go
  39. 53
      godo/ai/types/download.go
  40. 2
      godo/ai/types/types.go
  41. 14
      godo/ai/vector/model/vectorlist.go
  42. 152
      godo/ai/vector/vector.go
  43. 2
      godo/cmd/main.go
  44. 8
      godo/go.mod
  45. 16
      godo/go.sum
  46. 31
      godo/libs/dir.go

0
godo/model/api/gitee.go → godo/ai/api/gitee.go

0
godo/model/api/openai.go → godo/ai/api/openai.go

47
godo/model/data.go → godo/ai/config/data.go

@ -1,15 +1,16 @@
package model
package config
import (
"encoding/json"
"fmt"
"godo/ai/types"
"godo/libs"
"os"
"path/filepath"
"sync"
)
var reqBodyMap = sync.Map{}
var ReqBodyMap = sync.Map{}
func GetConfigFile() (string, error) {
modelDir, err := libs.GetAppDir()
@ -36,7 +37,7 @@ func LoadConfig() error {
if err != nil {
return err
}
var reqBodies []ReqBody
var reqBodies []types.ReqBody
content, err := os.ReadFile(filePath)
if err != nil {
return err
@ -46,9 +47,9 @@ func LoadConfig() error {
return err
}
for _, reqBody := range reqBodies {
reqBodyMap.Store(reqBody.Model, reqBody)
ReqBodyMap.Store(reqBody.Model, reqBody)
}
//log.Printf("Load config file success %v", reqBodyMap)
//log.Printf("Load config file success %v", ReqBodyMap)
return nil
}
@ -58,9 +59,9 @@ func SaveConfig() error {
if err != nil {
return err
}
var reqBodies []ReqBody
reqBodyMap.Range(func(key, value interface{}) bool {
rb := value.(ReqBody)
var reqBodies []types.ReqBody
ReqBodyMap.Range(func(key, value interface{}) bool {
rb := value.(types.ReqBody)
reqBodies = append(reqBodies, rb)
return true
})
@ -77,20 +78,20 @@ func SaveConfig() error {
}
return nil
}
func GetModel(Model string) (ReqBody, bool) {
value, ok := reqBodyMap.Load(Model)
func GetModel(Model string) (types.ReqBody, bool) {
value, ok := ReqBodyMap.Load(Model)
if ok {
return value.(ReqBody), true
return value.(types.ReqBody), true
}
return ReqBody{}, false
return types.ReqBody{}, false
}
func ExistModel(Model string) bool {
_, exists := reqBodyMap.Load(Model)
_, exists := ReqBodyMap.Load(Model)
return exists
}
func SetModel(reqBody ReqBody) error {
func SetModel(reqBody types.ReqBody) error {
reqBodyMap.Store(reqBody.Model, reqBody)
ReqBodyMap.Store(reqBody.Model, reqBody)
//log.Println("=====SetModel", reqBody.Model)
if err := SaveConfig(); err != nil {
@ -100,13 +101,13 @@ func SetModel(reqBody ReqBody) error {
return nil
}
func UpdateModel(reqBody ReqBody) error {
_, loaded := reqBodyMap.Load(reqBody.Model)
func UpdateModel(reqBody types.ReqBody) error {
_, loaded := ReqBodyMap.Load(reqBody.Model)
if !loaded {
return fmt.Errorf("model directory %s not found", reqBody.Model)
}
reqBodyMap.Store(reqBody.Model, reqBody)
ReqBodyMap.Store(reqBody.Model, reqBody)
if err := SaveConfig(); err != nil {
return fmt.Errorf("failed to save updated model configuration: %w", err)
}
@ -114,13 +115,13 @@ func UpdateModel(reqBody ReqBody) error {
return nil
}
func AddModel(Model string, reqBody ReqBody) error {
_, loaded := reqBodyMap.Load(Model)
func AddModel(Model string, reqBody types.ReqBody) error {
_, loaded := ReqBodyMap.Load(Model)
if loaded {
return fmt.Errorf("model directory %s already exists", Model)
}
reqBodyMap.Store(Model, reqBody)
ReqBodyMap.Store(Model, reqBody)
if err := SaveConfig(); err != nil {
return fmt.Errorf("failed to save new model configuration: %w", err)
}
@ -129,9 +130,9 @@ func AddModel(Model string, reqBody ReqBody) error {
}
func DeleteModel(Model string) error {
_, loaded := reqBodyMap.Load(Model)
_, loaded := ReqBodyMap.Load(Model)
if loaded {
reqBodyMap.Delete(Model)
ReqBodyMap.Delete(Model)
}
if err := SaveConfig(); err != nil {
return fmt.Errorf("failed to delete model configuration: %w", err)

45
godo/model/libs.go → godo/ai/config/libs.go

@ -1,8 +1,9 @@
package model
package config
import (
"encoding/json"
"fmt"
"godo/ai/types"
"godo/libs"
"net/http"
"net/url"
@ -52,22 +53,12 @@ func GetModelPath(urls string, model string, reqType string) (string, error) {
}
// 构建完整的文件路径
filePath := filepath.Join(modelDir, fileName)
if reqType == "local" {
dir := GetOllamaModelDir()
if strings.Contains(fileName, "sha256-") && len(fileName) == 71 {
filePath = filepath.Join(dir, "blobs", fileName)
//log.Printf("====filePath1: %s", filePath)
} else {
opName := getOpName(model)
filePath = filepath.Join(dir, "manifests", opName.Space, opName.LibPath, opName.Name, opName.Tag)
//log.Printf("====filePath2: %s", filePath)
}
}
// if reqType == "local" {
// }
return filePath, nil
}
func Var(key string) string {
return strings.Trim(strings.TrimSpace(os.Getenv(key)), "\"'")
}
func GetHfModelDir() (string, error) {
aiDir, ok := libs.GetConfig("aiDir")
if ok {
@ -79,33 +70,15 @@ func GetHfModelDir() (string, error) {
}
func GetOllamaModelDir() string {
if s := Var("OLLAMA_MODELS"); s != "" {
return s
}
home, _ := os.UserHomeDir()
return filepath.Join(home, ".ollama", "models")
}
func GetOllamaUrl() string {
if s := strings.TrimSpace(Var("OLLAMA_HOST")); s != "" {
return s
}
ollamaUrl, ok := libs.GetConfig("ollamaUrl")
if ok {
return ollamaUrl.(string)
} else {
return "http://localhost:11434"
}
}
func ReplaceModelName(modelName string) string {
reg := regexp.MustCompile(`[/\s:]`)
return reg.ReplaceAllString(modelName, "")
}
// ModelConfigFromRequest 解析HTTP请求中的JSON数据并填充ModelConfig,如果请求中没有'modelconfig'键或解析出错,则返回一个空的ModelConfig
func ModelConfigFromRequest(r *http.Request) ModelConfig {
func ModelConfigFromRequest(r *http.Request) types.ModelConfig {
// 初始化一个空的ModelConfig
var config ModelConfig
var config types.ModelConfig
// 尝试解析请求体中的JSON数据
var requestData map[string]interface{}
@ -122,7 +95,7 @@ func ModelConfigFromRequest(r *http.Request) ModelConfig {
}
// 如果没有'modelconfig'键或者解析出错,直接返回一个空的ModelConfig
return ModelConfig{}
return types.ModelConfig{}
}
func getIntInfo(val interface{}) int64 {
if val, ok := val.(float64); ok {

2
godo/convert/doc.go → godo/ai/convert/doc.go

@ -21,7 +21,7 @@ import (
"bytes"
"io"
"godo/convert/doc"
"godo/ai/convert/doc"
)
// ConvertDoc converts an MS Word .doc to text.

0
godo/convert/doc/clx.go → godo/ai/convert/doc/clx.go

0
godo/convert/doc/doc.go → godo/ai/convert/doc/doc.go

0
godo/convert/doc/fib.go → godo/ai/convert/doc/fib.go

0
godo/convert/doc/plcFld.go → godo/ai/convert/doc/plcFld.go

0
godo/convert/doc/utf16.go → godo/ai/convert/doc/utf16.go

0
godo/convert/docx.go → godo/ai/convert/docx.go

2
godo/convert/epub.go → godo/ai/convert/epub.go

@ -27,7 +27,7 @@ import (
"os"
"path"
"godo/convert/libs"
"godo/ai/convert/libs"
)
const containerPath = "META-INF/container.xml"

0
godo/convert/find.go → godo/ai/convert/find.go

0
godo/convert/html.go → godo/ai/convert/html.go

0
godo/convert/http.go → godo/ai/convert/http.go

2
godo/convert/image.go → godo/ai/convert/image.go

@ -20,7 +20,7 @@ package convert
import (
"io"
"godo/convert/libs"
"godo/ai/convert/libs"
)
func ConvertImage(r io.Reader) (string, error) {

2
godo/convert/img.go → godo/ai/convert/img.go

@ -20,7 +20,7 @@ package convert
import (
"crypto/md5"
"fmt"
lib "godo/convert/libs"
lib "godo/ai/convert/libs"
"godo/libs"
"io"
"log"

0
godo/convert/libs/file.go → godo/ai/convert/libs/file.go

0
godo/convert/libs/kind.go → godo/ai/convert/libs/kind.go

0
godo/convert/libs/rapid.go → godo/ai/convert/libs/rapid.go

0
godo/convert/libs/xpdf.go → godo/ai/convert/libs/xpdf.go

0
godo/convert/main.go → godo/ai/convert/main.go

0
godo/convert/main_test.go → godo/ai/convert/main_test.go

0
godo/convert/md.go → godo/ai/convert/md.go

0
godo/convert/odt.go → godo/ai/convert/odt.go

2
godo/convert/pdf.go → godo/ai/convert/pdf.go

@ -20,7 +20,7 @@ package convert
import (
"io"
"godo/convert/libs"
"godo/ai/convert/libs"
)
func ConvertPDF(r io.Reader) (string, error) {

0
godo/convert/pptx.go → godo/ai/convert/pptx.go

0
godo/convert/rtf.go → godo/ai/convert/rtf.go

0
godo/convert/tidy.go → godo/ai/convert/tidy.go

0
godo/convert/txt.go → godo/ai/convert/txt.go

0
godo/convert/url.go → godo/ai/convert/url.go

2
godo/convert/xlsx.go → godo/ai/convert/xlsx.go

@ -21,7 +21,7 @@ import (
"io"
"strings"
"godo/convert/libs"
"godo/ai/convert/libs"
"github.com/pbnjay/grate"
_ "github.com/pbnjay/grate/simple" // tsv and csv support

0
godo/convert/xml.go → godo/ai/convert/xml.go

2
godo/model/chat.go → godo/ai/server/chat.go

@ -1,4 +1,4 @@
package model
package server
import (
"encoding/json"

2
godo/model/common.go → godo/ai/server/common.go

@ -1,4 +1,4 @@
package model
package server
import (
"bytes"

24
godo/model/down.go → godo/ai/server/down.go

@ -1,9 +1,11 @@
package model
package server
import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"godo/ai/config"
"godo/ai/types"
"godo/libs"
"io"
"log"
@ -35,18 +37,18 @@ func noticeSuccess(w http.ResponseWriter) {
}
func Download(w http.ResponseWriter, r *http.Request) {
reqBody := ReqBody{}
reqBody := types.ReqBody{}
err := json.NewDecoder(r.Body).Decode(&reqBody)
if err != nil {
libs.ErrorMsg(w, "first Decode request body error:"+err.Error())
return
}
err = LoadConfig()
err = config.LoadConfig()
if err != nil {
libs.ErrorMsg(w, "Load config error")
return
}
_, exitsModel := GetModel(reqBody.Model)
_, exitsModel := config.GetModel(reqBody.Model)
if exitsModel {
noticeSuccess(w)
return
@ -71,7 +73,7 @@ func Download(w http.ResponseWriter, r *http.Request) {
paths = append(paths, urls)
continue
}
filePath, err := GetModelPath(urls, reqBody.Model, reqBody.Type)
filePath, err := config.GetModelPath(urls, reqBody.Model, reqBody.Type)
//log.Printf("filePath is %s", filePath)
if err != nil {
libs.ErrorMsg(w, "Get model path error")
@ -127,7 +129,7 @@ func Download(w http.ResponseWriter, r *http.Request) {
// reqBody.Paths = []string{}
}
if err := SetModel(reqBody); err != nil {
if err := config.SetModel(reqBody); err != nil {
libs.ErrorMsg(w, "Set model error")
return
}
@ -154,7 +156,7 @@ func trackProgress(w http.ResponseWriter, resp *grab.Response, md5url string) {
for {
select {
case <-ticker.C:
fp := FileProgress{
fp := types.FileProgress{
Progress: resp.Progress(),
IsFinished: resp.IsComplete(),
Total: resp.Size(),
@ -202,18 +204,18 @@ func delUrls(reqUrl []string) {
}
}
func DeleteFileHandle(w http.ResponseWriter, r *http.Request) {
var reqBody ReqBody
var reqBody types.ReqBody
err := json.NewDecoder(r.Body).Decode(&reqBody)
if err != nil {
libs.ErrorMsg(w, "Decode request body error: ")
return
}
err = LoadConfig()
err = config.LoadConfig()
if err != nil {
libs.ErrorMsg(w, "Load config error: ")
return
}
if err := DeleteModel(reqBody.Model); err != nil {
if err := config.DeleteModel(reqBody.Model); err != nil {
libs.ErrorMsg(w, "Error deleting model")
return
}
@ -228,7 +230,7 @@ func DeleteFileHandle(w http.ResponseWriter, r *http.Request) {
// 尝试删除目录,注意这会递归删除目录下的所有内容
//dirPath := filepath.Dir(filePath)
dirPath, err := GetModelDir(reqBody.Model)
dirPath, err := config.GetModelDir(reqBody.Model)
if err != nil {
libs.ErrorMsg(w, "GetModelDir error")
return

135
godo/model/ollama.go → godo/ai/server/ollama.go

@ -1,10 +1,12 @@
package model
package server
import (
"bytes"
"crypto/sha256"
"encoding/json"
"fmt"
"godo/ai/config"
"godo/ai/types"
"godo/libs"
"io"
"log"
@ -17,58 +19,6 @@ import (
"time"
)
type OllamaModelsList struct {
Models []OllamaModelsInfo `json:"models"`
}
type OllamaDetails struct {
ParameterSize string `json:"parameter_size"`
QuantizationLevel string `json:"quantization_level"`
}
type OllamaModelsInfo struct {
Model string `json:"model"`
Details OllamaDetails `json:"details"`
Size int64 `json:"size"`
}
type OllamaModelDetail struct {
Parameters string `json:"parameters"`
Template string `json:"template"`
Details map[string]interface{} `json:"details"`
ModelInfo map[string]interface{} `json:"model_info"`
}
// type ResModelInfo struct {
// Parameters string `json:"parameters"`
// Template string `json:"template"`
// ContextLength int64 `json:"context_length"`
// EmbeddingLength int64 `json:"embedding_length"`
// Size string `json:"size"`
// Quant string `json:"quant"`
// Desk string `json:"desk"`
// Cpu string `json:"cpu"`
// Gpu string `json:"gpu"`
// }
type Layer struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size int64 `json:"size"`
From string `json:"from,omitempty"`
status string
}
type ManifestV2 struct {
SchemaVersion int `json:"schemaVersion"`
MediaType string `json:"mediaType"`
Config *Layer `json:"config"`
Layers []*Layer `json:"layers"`
}
type OmodelPath struct {
Space string
LibPath string
Name string
Tag string
}
const (
KB = 1 << (10 * iota)
MB
@ -116,8 +66,8 @@ func extractParameterSize(sizeStr string, model string) (float64, bool) {
return 0, false
}
func parseOllamaInfo(info OllamaModelsInfo) ModelInfo {
res := ModelInfo{
func parseOllamaInfo(info types.OllamaModelsInfo) types.ModelInfo {
res := types.ModelInfo{
Size: humanReadableSize(info.Size),
Quant: info.Details.QuantizationLevel,
}
@ -143,9 +93,9 @@ func parseOllamaInfo(info OllamaModelsInfo) ModelInfo {
return res
}
func getOllamaModels() ([]OllamaModelsInfo, error) {
func getOllamaModels() ([]types.OllamaModelsInfo, error) {
req, err := http.Get(GetOllamaUrl() + "/api/tags")
res := []OllamaModelsInfo{}
res := []types.OllamaModelsInfo{}
if err != nil {
return res, fmt.Errorf("failed to create request")
}
@ -155,7 +105,7 @@ func getOllamaModels() ([]OllamaModelsInfo, error) {
if err != nil {
return res, fmt.Errorf("failed to read response body")
}
rest := OllamaModelsList{}
rest := types.OllamaModelsList{}
if err := json.Unmarshal(body, &rest); err != nil {
return res, fmt.Errorf("failed to unmarshal response body")
}
@ -176,10 +126,10 @@ func refreshOllamaModels(r *http.Request) error {
if err != nil {
return fmt.Errorf("load ollama error: %v", err)
}
// 将modelList中的数据写入reqBodyMap
// 将modelList中的数据写入ReqBodyMap
for _, modelInfo := range modelList {
model := modelInfo.Model
if _, exists := reqBodyMap.Load(model); !exists {
if _, exists := config.ReqBodyMap.Load(model); !exists {
// 创建一个新的ReqBody对象并填充相关信息
oinfo := parseOllamaInfo(modelInfo)
details, err := getOllamaInfo(r, model)
@ -195,12 +145,12 @@ func refreshOllamaModels(r *http.Request) error {
log.Printf("Error parsing Manifests: %v", err)
continue
}
reqBody := ReqBody{
reqBody := types.ReqBody{
Model: model,
Status: "success",
CreatedAt: time.Now(),
}
reqBody.Info = ModelInfo{
reqBody.Info = types.ModelInfo{
Engine: "ollama",
From: "ollama",
Path: paths,
@ -215,13 +165,13 @@ func refreshOllamaModels(r *http.Request) error {
EmbeddingLength: embeddingLength,
}
// 将新的ReqBody对象写入reqBodyMap
reqBodyMap.Store(model, reqBody)
// 将新的ReqBody对象写入ReqBodyMap
config.ReqBodyMap.Store(model, reqBody)
}
}
return nil
}
func setOllamaInfo(w http.ResponseWriter, r *http.Request, reqBody ReqBody) {
func setOllamaInfo(w http.ResponseWriter, r *http.Request, reqBody types.ReqBody) {
model := reqBody.Model
postQuery := map[string]interface{}{
"model": model,
@ -256,7 +206,7 @@ func setOllamaInfo(w http.ResponseWriter, r *http.Request, reqBody ReqBody) {
continue
}
reqBody.Info = ModelInfo{
reqBody.Info = types.ModelInfo{
Engine: reqBody.Info.Engine,
From: reqBody.Info.From,
Path: paths,
@ -273,7 +223,7 @@ func setOllamaInfo(w http.ResponseWriter, r *http.Request, reqBody ReqBody) {
//reqBody.Paths = paths
reqBody.Status = "success"
reqBody.CreatedAt = time.Now()
if err := SetModel(reqBody); err != nil {
if err := config.SetModel(reqBody); err != nil {
libs.ErrorMsg(w, "Set model error")
return
}
@ -295,11 +245,11 @@ func convertInt(data map[string]interface{}, str string) int {
}
return res
}
func getOllamaInfo(r *http.Request, model string) (OllamaModelDetail, error) {
func getOllamaInfo(r *http.Request, model string) (types.OllamaModelDetail, error) {
infoQuery := map[string]interface{}{
"name": model,
}
res := OllamaModelDetail{}
res := types.OllamaModelDetail{}
url := GetOllamaUrl() + "/api/show"
payloadBytes, err := json.Marshal(infoQuery)
if err != nil {
@ -329,7 +279,7 @@ func getOllamaInfo(r *http.Request, model string) (OllamaModelDetail, error) {
}
return res, nil
}
func getOpName(model string) OmodelPath {
func GetOpName(model string) types.OmodelPath {
libPath := "library"
modelName := model
modelTags := "latest"
@ -344,7 +294,7 @@ func getOpName(model string) OmodelPath {
libPath = names[0]
modelName = names[1]
}
return OmodelPath{
return types.OmodelPath{
Space: "registry.ollama.ai",
LibPath: libPath,
Name: modelName,
@ -353,14 +303,14 @@ func getOpName(model string) OmodelPath {
}
func getManifests(model string) ([]string, error) {
res := []string{}
opName := getOpName(model)
opName := GetOpName(model)
modelsDir := GetOllamaModelDir()
manifestsFile := filepath.Join(modelsDir, "manifests", opName.Space, opName.LibPath, opName.Name, opName.Tag)
if !libs.PathExists(manifestsFile) {
return res, fmt.Errorf("failed to get manifests file: %s", manifestsFile)
}
res = append(res, manifestsFile)
var manifest ManifestV2
var manifest types.ManifestV2
f, err := os.Open(manifestsFile)
if err != nil {
return res, err
@ -401,7 +351,7 @@ func GetBlobsPath(digest string) (string, error) {
return path, nil
}
func ConvertOllama(w http.ResponseWriter, r *http.Request, req ReqBody) {
func ConvertOllama(w http.ResponseWriter, r *http.Request, req types.ReqBody) {
modelFile := "FROM " + req.Info.Path[0] + "\n"
modelFile += `TEMPLATE """` + req.Info.Template + `"""`
if req.Info.Parameters != "" {
@ -417,7 +367,7 @@ func ConvertOllama(w http.ResponseWriter, r *http.Request, req ReqBody) {
"modelfile": modelFile,
}
ForwardHandler(w, r, postParams, url, "POST")
modelDir, err := GetModelDir(req.Model)
modelDir, err := config.GetModelDir(req.Model)
if err != nil {
libs.ErrorMsg(w, "GetModelDir")
return
@ -434,3 +384,38 @@ func ConvertOllama(w http.ResponseWriter, r *http.Request, req ReqBody) {
return
}
}
func Var(key string) string {
return strings.Trim(strings.TrimSpace(os.Getenv(key)), "\"'")
}
func GetOllamaModelDir() string {
if s := Var("OLLAMA_MODELS"); s != "" {
return s
}
home, _ := os.UserHomeDir()
return filepath.Join(home, ".ollama", "models")
}
func GetOllamaUrl() string {
if s := strings.TrimSpace(Var("OLLAMA_HOST")); s != "" {
return s
}
ollamaUrl, ok := libs.GetConfig("ollamaUrl")
if ok {
return ollamaUrl.(string)
} else {
return "http://localhost:11434"
}
}
func GetModelDir(fileName string, model string) string {
var filePath string
dir := GetOllamaModelDir()
if strings.Contains(fileName, "sha256-") && len(fileName) == 71 {
filePath = filepath.Join(dir, "blobs", fileName)
//log.Printf("====filePath1: %s", filePath)
} else {
opName := GetOpName(model)
filePath = filepath.Join(dir, "manifests", opName.Space, opName.LibPath, opName.Name, opName.Tag)
//log.Printf("====filePath2: %s", filePath)
}
return filePath
}

20
godo/model/server.go → godo/ai/server/server.go

@ -1,8 +1,10 @@
package model
package server
import (
"encoding/json"
"fmt"
"godo/ai/config"
"godo/ai/types"
"godo/libs"
"io"
"log"
@ -13,14 +15,14 @@ import (
)
func Tagshandler(w http.ResponseWriter, r *http.Request) {
err := LoadConfig()
err := config.LoadConfig()
if err != nil {
libs.ErrorMsg(w, "Load config error")
return
}
var reqBodies []ReqBody
reqBodyMap.Range(func(key, value interface{}) bool {
rb, ok := value.(ReqBody)
var reqBodies []types.ReqBody
config.ReqBodyMap.Range(func(key, value interface{}) bool {
rb, ok := value.(types.ReqBody)
if ok {
reqBodies = append(reqBodies, rb)
}
@ -40,7 +42,7 @@ func Tagshandler(w http.ResponseWriter, r *http.Request) {
}
}
func ShowHandler(w http.ResponseWriter, r *http.Request) {
err := LoadConfig()
err := config.LoadConfig()
if err != nil {
libs.ErrorMsg(w, "Load config error")
return
@ -51,9 +53,9 @@ func ShowHandler(w http.ResponseWriter, r *http.Request) {
return
}
//log.Printf("ShowHandler: %s", model)
var reqBodies ReqBody
reqBodyMap.Range(func(key, value interface{}) bool {
rb, ok := value.(ReqBody)
var reqBodies types.ReqBody
config.ReqBodyMap.Range(func(key, value interface{}) bool {
rb, ok := value.(types.ReqBody)
if ok && rb.Model == model {
reqBodies = rb
return false

53
godo/ai/types/download.go

@ -0,0 +1,53 @@
package types
type OllamaModelsList struct {
Models []OllamaModelsInfo `json:"models"`
}
type OllamaDetails struct {
ParameterSize string `json:"parameter_size"`
QuantizationLevel string `json:"quantization_level"`
}
type OllamaModelsInfo struct {
Model string `json:"model"`
Details OllamaDetails `json:"details"`
Size int64 `json:"size"`
}
type OllamaModelDetail struct {
Parameters string `json:"parameters"`
Template string `json:"template"`
Details map[string]interface{} `json:"details"`
ModelInfo map[string]interface{} `json:"model_info"`
}
// type ResModelInfo struct {
// Parameters string `json:"parameters"`
// Template string `json:"template"`
// ContextLength int64 `json:"context_length"`
// EmbeddingLength int64 `json:"embedding_length"`
// Size string `json:"size"`
// Quant string `json:"quant"`
// Desk string `json:"desk"`
// Cpu string `json:"cpu"`
// Gpu string `json:"gpu"`
// }
type Layer struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size int64 `json:"size"`
From string `json:"from,omitempty"`
status string
}
type ManifestV2 struct {
SchemaVersion int `json:"schemaVersion"`
MediaType string `json:"mediaType"`
Config *Layer `json:"config"`
Layers []*Layer `json:"layers"`
}
type OmodelPath struct {
Space string
LibPath string
Name string
Tag string
}

2
godo/model/types.go → godo/ai/types/types.go

@ -1,4 +1,4 @@
package model
package types
import "time"

14
godo/ai/vector/model/vectorlist.go

@ -0,0 +1,14 @@
package model
import "gorm.io/gorm"
type VectorList struct {
gorm.Model
Name string `json:"name"`
FilePath string `json:"file_path"`
DbPath string `json:"db_path"`
Engine string `json:"engine"`
EmbeddingUrl string `json:"embedding_url"`
EmbeddingApiKey string `json:"embedding_api_key"`
EmbeddingModel string `json:"embedding_model"`
}

152
godo/ai/vector/vector.go

@ -0,0 +1,152 @@
package vector
import (
"database/sql"
"fmt"
"godo/ai/vector/model"
"godo/libs"
"os"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/gormlite"
"gorm.io/gorm"
)
var vectorListDb *gorm.DB
var dbPathToSqlDB = make(map[string]*sql.DB)
func init() {
dbPath := libs.GetVectorDb()
db, err := gorm.Open(gormlite.Open(dbPath), &gorm.Config{})
if err != nil {
return
}
vectorListDb = db
}
// CreateVector 创建一个新的 VectorList 记录
func CreateVector(name string, filePath string) (*gorm.DB, error) {
dbPath := libs.GetVectorPath(name)
var list model.VectorList
// 检查是否已经存在同名的 VectorList
result := vectorListDb.Where("name = ?", name).First(&list)
if result.Error == nil {
return nil, fmt.Errorf("vector list with the same name already exists")
}
// 创建新的 VectorList 记录
newList := model.VectorList{
Name: name,
FilePath: filePath, // 根据实际情况设置文件路径
DbPath: dbPath,
}
result = vectorListDb.Create(&newList)
if result.Error != nil {
return nil, result.Error
}
return vectorListDb, nil
}
// DeleteVector 删除指定名称的 VectorList 记录
func DeleteVector(name string) error {
result := vectorListDb.Where("name = ?", name).Delete(&model.VectorList{})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return fmt.Errorf("vector list not found")
}
// 关闭数据库连接
err := CloseVectorDb(name)
if err != nil {
return fmt.Errorf("failed to close vector database: %w", err)
}
// 删除数据库文件
dbPath := libs.GetVectorPath(name)
if libs.PathExists(dbPath) {
err := os.Remove(dbPath)
if err != nil {
return fmt.Errorf("failed to delete database file: %w", err)
}
}
return nil
}
// CloseVectorDb 关闭指定名称的 Vector 数据库连接
func CloseVectorDb(name string) error {
dbPath := libs.GetVectorPath(name)
if !libs.PathExists(dbPath) {
return nil
}
sqlDB, exists := dbPathToSqlDB[dbPath]
if !exists {
return fmt.Errorf("no database connection found for path: %s", dbPath)
}
err := sqlDB.Close()
if err != nil {
return fmt.Errorf("failed to close database connection: %w", err)
}
delete(dbPathToSqlDB, name)
return sqlDB.Close()
}
// RenameVectorDb 更改指定名称的 VectorList 的数据库名称
func RenameVectorDb(oldName string, newName string) error {
// 1. 检查并关闭旧的数据库连接(如果已打开)
err := CloseVectorDb(oldName)
if err != nil {
return fmt.Errorf("failed to close vector database: %w", err)
}
// 2. 获取旧的 VectorList 记录
var oldList model.VectorList
result := vectorListDb.Where("name = ?", oldName).First(&oldList)
if result.Error != nil {
return fmt.Errorf("failed to find old vector list: %w", result.Error)
}
//3. 删除旧的数据库文件
oldDbPath := libs.GetVectorPath(oldName)
// 4. 构建新的 DbPath
newDbPath := libs.GetVectorPath(newName)
if libs.PathExists(oldDbPath) {
err := os.Rename(oldDbPath, newDbPath)
if err != nil {
return fmt.Errorf("failed to move old database file to new location: %w", err)
}
}
// 5. 更新 VectorList 记录中的 DbPath 和 Name
oldList.Name = newName
oldList.DbPath = newDbPath
result = vectorListDb.Save(&oldList)
if result.Error != nil {
return fmt.Errorf("failed to update vector list: %w", result.Error)
}
return nil
}
// UpdateVector 更新指定名称的 VectorList 记录
func UpdateVector(name string, updates map[string]interface{}) error {
result := vectorListDb.Model(&model.VectorList{}).Where("name = ?", name).Updates(updates)
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return fmt.Errorf("vector list not found")
}
return nil
}
func GetVectorList() []model.VectorList {
var vectorList []model.VectorList
vectorListDb.Find(&vectorList)
return vectorList
}

2
godo/cmd/main.go

@ -20,11 +20,11 @@ package cmd
import (
"context"
model "godo/ai/server"
"godo/deps"
"godo/files"
"godo/libs"
"godo/localchat"
"godo/model"
"godo/store"
"godo/sys"
"godo/webdav"

8
godo/go.mod

@ -12,24 +12,32 @@ require (
github.com/gorilla/mux v1.8.1
github.com/mattetti/filebuffer v1.0.1
github.com/minio/selfupdate v0.6.0
github.com/ncruces/go-sqlite3 v0.20.3
github.com/ncruces/go-sqlite3/gormlite v0.20.3
github.com/pbnjay/grate v0.0.0-20231006022435-3f8e65d74a14
github.com/richardlehane/mscfb v1.0.4
github.com/shirou/gopsutil v3.21.11+incompatible
golang.org/x/net v0.21.0
gorm.io/gorm v1.25.12
jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056
)
require (
aead.dev/minisign v0.2.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/richardlehane/msoleps v1.0.1 // indirect
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/stretchr/testify v1.9.0 // indirect
github.com/tetratelabs/wazero v1.8.2 // indirect
github.com/tklauser/go-sysconf v0.3.14 // indirect
github.com/tklauser/numcpus v0.8.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
golang.org/x/crypto v0.29.0 // indirect
golang.org/x/sys v0.27.0 // indirect
golang.org/x/text v0.20.0 // indirect
)

16
godo/go.sum

@ -14,12 +14,22 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/mattetti/filebuffer v1.0.1 h1:gG7pyfnSIZCxdoKq+cPa8T0hhYtD9NxCdI4D7PTjRLM=
github.com/mattetti/filebuffer v1.0.1/go.mod h1:YdMURNDOttIiruleeVr6f56OrMc+MydEnTcXwtkxNVs=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
github.com/ncruces/go-sqlite3 v0.20.3 h1:+4G4uEqOeusF0yRuQVUl9fuoEebUolwQSnBUjYBLYIw=
github.com/ncruces/go-sqlite3 v0.20.3/go.mod h1:ojLIAB243gtz68Eo283Ps+k9PyR3dvzS+9/RgId4+AA=
github.com/ncruces/go-sqlite3/gormlite v0.20.3 h1:bpnLMMhWFoiY6EHF52pc3Czl8tJCSwEG1a19WzIa/Co=
github.com/ncruces/go-sqlite3/gormlite v0.20.3/go.mod h1:wxYIO+HouPGhQ9SmnxsStnrMLs8xk6vVkVTwfG9coHo=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/pbnjay/grate v0.0.0-20231006022435-3f8e65d74a14 h1:ZfXdW7GIVZT3Z9oejLJ+GHrrQv/ezU2Bwqn0BF37s4g=
@ -36,6 +46,8 @@ github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cma
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY=
@ -64,8 +76,12 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 h1:6YFJoB+0fUH6X3xU/G2tQqCYg+PkGtnZ5nMR5rpw72g=
jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:OxvTsCwKosqQ1q7B+8FwXqg4rKZ/UG9dUW+g/VL2xH4=

31
godo/libs/dir.go

@ -19,6 +19,8 @@
package libs
import (
"crypto/md5"
"encoding/hex"
"fmt"
"os"
"path/filepath"
@ -122,6 +124,35 @@ func GetDataDir() string {
}
return staticPath
}
func GetDbDir() string {
homeDir := GetDataDir()
dbPath := filepath.Join(homeDir, "db", "vector")
if !PathExists(dbPath) {
os.MkdirAll(dbPath, 0755)
}
return dbPath
}
func GetVectorDb() string {
homeDir := GetDataDir()
dbPath := filepath.Join(homeDir, "db")
if !PathExists(dbPath) {
os.MkdirAll(dbPath, 0755)
}
return filepath.Join(dbPath, "vector.db")
}
func GetVectorDbName(name string) string {
hasher := md5.New()
hasher.Write([]byte(name))
hashBytes := hasher.Sum(nil)
// 将哈希值转换为十六进制字符串
return hex.EncodeToString(hashBytes)
}
func GetVectorPath(name string) string {
hashHex := GetVectorDbName(name)
dbDir := GetDbDir()
return filepath.Join(dbDir, "vector", hashHex+".db")
}
func GetCacheDir() string {
homeDir := GetDataDir()
cachePath := filepath.Join(homeDir, "cache")

Loading…
Cancel
Save