Browse Source

add onlyoffice

master
godo 6 months ago
parent
commit
80f2229a49
  1. 13
      frontend/package-lock.json
  2. 1
      frontend/package.json
  3. 1
      frontend/src/components/desktop/LockDesktop.vue
  4. 57
      frontend/src/components/setting/SetSystem.vue
  5. 101
      frontend/src/components/window/OnlyOffice.vue
  6. 12
      frontend/src/components/window/WindowInner.vue
  7. 12
      frontend/src/system/config.ts
  8. 87
      godo/ai/vector/document.go
  9. 1
      godo/ai/vector/model/vectorlist.go
  10. 72
      godo/ai/vector/vector.go
  11. 48
      godo/files/pwd.go
  12. 250
      godo/office/document.go
  13. 244
      godo/office/splittext.go
  14. 171
      godo/office/totxt.go
  15. 1
      godo/office/types.go
  16. 29
      packages/pptx/README.md
  17. 19
      packages/pptx/package-lock.json
  18. 3
      packages/pptx/package.json

13
frontend/package-lock.json

@ -1,15 +1,16 @@
{
"name": "godoos",
"version": "1.0.2",
"version": "1.0.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "godoos",
"version": "1.0.2",
"version": "1.0.3",
"dependencies": {
"@element-plus/icons-vue": "^2.3.1",
"@liripeng/vue-audio-player": "^1.6.2",
"@onlyoffice/document-editor-vue": "^1.4.0",
"cherry-markdown": "^0.8.52",
"dexie": "^4.0.8",
"element-plus": "^2.7.7",
@ -644,6 +645,14 @@
"node": ">= 8"
}
},
"node_modules/@onlyoffice/document-editor-vue": {
"version": "1.4.0",
"resolved": "https://registry.npmmirror.com/@onlyoffice/document-editor-vue/-/document-editor-vue-1.4.0.tgz",
"integrity": "sha512-Fg5gSc1zF6bmpRapUd7rMpm7kEDF7mQIHQKfcsfJcILdFX9bwIhnkXEucETEA9zdt92nWMS6qiAgVeT61TdCyw==",
"peerDependencies": {
"vue": "^3.0.0"
}
},
"node_modules/@parcel/watcher": {
"version": "2.5.0",
"resolved": "https://registry.npmmirror.com/@parcel/watcher/-/watcher-2.5.0.tgz",

1
frontend/package.json

@ -11,6 +11,7 @@
"dependencies": {
"@element-plus/icons-vue": "^2.3.1",
"@liripeng/vue-audio-player": "^1.6.2",
"@onlyoffice/document-editor-vue": "^1.4.0",
"cherry-markdown": "^0.8.52",
"dexie": "^4.0.8",
"element-plus": "^2.7.7",

1
frontend/src/components/desktop/LockDesktop.vue

@ -72,7 +72,6 @@
>
<a
href="#"
v-if="config.userType !== 'member'"
@click.prevent="toggleUserSwitch"
>切换角色</a
>

57
frontend/src/components/setting/SetSystem.vue

@ -63,6 +63,31 @@
</div>
<div v-if="2 === activeIndex">
<div class="setting-item" style="margin-top: 60px">
<label>编辑器类型</label>
<el-select v-model="config.editorType">
<el-option v-for="(item, key) in editorType" :key="key" :label="item.title"
:value="item.value" />
</el-select>
</div>
<template v-if="config.editorType === 'onlyoffice'">
<div class="setting-item">
<label>地址</label>
<el-input v-model="config.onlyoffice.url" placeholder="https://godoos.com/onlyoffice 不要加斜杠" />
</div>
<div class="setting-item">
<label>私钥</label>
<el-input v-model="config.onlyoffice.sceret" />
</div>
</template>
<div class="setting-item">
<label></label>
<el-button @click="submitEditInfo" type="primary">
{{ t("confirm") }}
</el-button>
</div>
</div>
<div v-if="3 === activeIndex">
<div class="setting-item">
<h1 class="setting-title">备份</h1>
</div>
@ -112,7 +137,7 @@
</el-button>
</div>
</div>
<div v-if="3 === activeIndex" class="setting-area">
<div v-if="4 === activeIndex" class="setting-area">
<SetFilePwd v-if="config.userType === 'person'"></SetFilePwd>
</div>
</div>
@ -155,7 +180,17 @@ const storeList = [
},
];
const items = ["用户角色", "存储配置", "备份还原", "文件密码箱"];
const items = ["用户角色", "存储配置", "编辑器类型", "备份还原", "文件密码箱"];
const editorType = [
{
title: "系统默认",
value: "local",
},
{
title: "OnlyOffice",
value: "onlyoffice",
},
];
const urlRegex = /^(https?:\/\/)/;
const userTypes = [
{
@ -181,7 +216,7 @@ function selectFile() {
function submitOsInfo() {
const saveData = toRaw(config.value);
if (saveData.storeType === "local") {
const postData = parseData(saveData);
const postData = parseData(saveData);
const postUrl = config.value.apiUrl + "/system/setting";
fetch(postUrl, {
method: "POST",
@ -260,13 +295,21 @@ function submitOsInfo() {
});
}
}
function submitEditInfo() {
const saveData = toRaw(config.value);
setSystemConfig(saveData);
Dialog.showMessageBox({
message: "保存成功",
type: "success",
});
return;
}
function parseData(saveData: any) {
let postData = []
if (saveData.storePath !== "") {
postData.push({ name: "osPath", value: saveData.storePath })
}
if (saveData.netPort != "" && saveData.netPort != "56780" && !isNaN(saveData.netPort) && saveData.netPort*1 > 0 && saveData.netPort*1 < 65535) {
postData.push({ name: "osPath", value: saveData.storePath })
}
if (saveData.netPort != "" && saveData.netPort != "56780" && !isNaN(saveData.netPort) && saveData.netPort * 1 > 0 && saveData.netPort * 1 < 65535) {
postData.push({
name: "netPort",
value: saveData.netPort,

101
frontend/src/components/window/OnlyOffice.vue

@ -0,0 +1,101 @@
<template>
<DocumentEditor id="docEditor" documentServerUrl="http://127.0.0.1:8000/" :config="editorConfig"
:events_onDocumentReady="onDocumentReady" :onLoadComponentError="onLoadComponentError" />
</template>
<script lang="ts" setup>
import { DocumentEditor } from "@onlyoffice/document-editor-vue";
import { getSystemConfig } from "@/system/config";
import { BrowserWindow, Dialog, Notify, System } from "@/system";
import { generateRandomString } from "@/util/common";
const config = getSystemConfig();
const sys: any = inject<System>("system");
const win: any = inject<BrowserWindow>("browserWindow");
const props = defineProps({
src: {
type: String,
default: "",
},
eventType: {
type: String,
default: "",
},
ext: {
type: String,
default: "md",
},
});
const editorConfig: any = ref({})
// async function fetchDocumentKey(path: string): Promise<string> {
// try {
// const response:any = await fetch(`${config.onlyoffice.url}/get-document-key?path=${path}`);
// console.log(response)
// const res = await response.json();
// return res.data.key;
// } catch (error) {
// console.error("Failed to fetch document key:", error);
// throw error;
// }
// }
onMounted(() => {
const path = win?.config?.path;
const uniqueKey = generateRandomString(12);
//const uniqueKey = fetchDocumentKey(path)
const readUrl = config.apiUrl + "/file/readfile?stream=1&path=" + path
editorConfig.value = {
document: {
fileType: "docx",
//key: "ojR1OasBPnlIwF9WA80AW4NTrIWqs9",
//"key": uniqueKey,
key: "docx" + Math.random(),
// "permissions": {
// "chat": true,
// "comment": true,
// "copy": true,
// "download": true,
// "edit": true,
// "fillForms": true,
// "modifyContentControl": true,
// "modifyFilter": true,
// "print": true,
// "review": true,
// "reviewGroups": null,
// "commentGroups": {},
// "userInfoGroups": null,
// "protect": true
// },
title: "Example Document Title.docx",
url: readUrl
},
documentType: "word",
editorConfig: {
callbackUrl: "https://example.com/url-to-callback.ashx",
// customization: {
// "anonymous": {
// request: true,
// label: "Guest",
// }
// },
}
}
})
const onDocumentReady = () => {
console.log("Document is loaded");
}
const onLoadComponentError = (errorCode: any, errorDescription: any) => {
switch (errorCode) {
case -1: // Unknown error loading component
console.log(errorDescription);
break;
case -2: // Error load DocsAPI from http://documentserver/
console.log(errorDescription);
break;
case -3: // DocsAPI is not defined
console.log(errorDescription);
break;
}
}
</script>

12
frontend/src/components/window/WindowInner.vue

@ -1,5 +1,9 @@
<template>
<IframeFile v-if="win.url" :src="win.url" :ext="win.ext" :eventType="win.eventType" />
<template v-if="win.url">
<IframeFile v-if="config.editorType =='local'" :src="win.url" :ext="win.ext" :eventType="win.eventType" />
<OnlyOffice v-else :src="win.url" :eventType="win.eventType" :ext="ext" />
</template>
<Suspense v-else>
<component v-if="win.content" :is="stepComponent(win.content)" :translateSavePath="translateSavePath" :componentID="win.windowInfo.componentID"></component>
<RouterView v-else />
@ -9,11 +13,17 @@
<script setup lang="ts">
import { useRouter } from "vue-router";
import { stepComponent } from "@/util/stepComponent";
import { getSystemConfig } from "@/system/config";
const config = getSystemConfig();
const router = useRouter();
const props = defineProps<{
win: any;
}>();
let ext = "txt"
const win = ref(props.win)
if(win.value.config.path){
ext = win.value.config.path.split('.').pop()
}
const translateSavePath = inject('translateSavePath')
if (props.win.path) {
router.push(props.win.path);

12
frontend/src/system/config.ts

@ -15,7 +15,7 @@ export const getSystemConfig = (ifset = false) => {
// 初始化配置对象的各项属性,若本地存储中已存在则不进行覆盖
if (!config.version) {
config.version = '1.0.3';
config.version = '1.0.4';
}
if (!config.isFirstRun) {
config.isFirstRun = false;
@ -30,6 +30,16 @@ export const getSystemConfig = (ifset = false) => {
if (!config.userType) {
config.userType = 'person'//如果是企业版请改为member
}
if(!config.editorType){
config.editorType = 'local'
}
if(!config.onlyoffice){
config.onlyoffice = {
url: '',
sceret: '',
}
}
if (!config.file) {
config.file = {
isPwd: 0,

87
godo/ai/vector/document.go

@ -0,0 +1,87 @@
package vector
import (
"godo/libs"
"godo/office"
"log"
"path"
"sync"
"github.com/fsnotify/fsnotify"
)
const numWorkers = 5 // 设置 worker 数量
func MonitorFolder(folderPath string) {
if !libs.PathExists(folderPath) {
return
}
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatalf("Failed to create watcher for folder %s: %v", folderPath, err)
}
defer watcher.Close()
fileQueue := make(chan string, 100) // 创建文件路径队列
var wg sync.WaitGroup
// 启动 worker goroutine
for i := 0; i < numWorkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for filePath := range fileQueue {
handleGodoosFile(filePath)
}
}()
}
done := make(chan bool)
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return
}
log.Println("event:", event)
if event.Op&(fsnotify.Create|fsnotify.Write) != 0 { // 监听创建和修改事件
baseName := path.Base(event.Name)
if baseName[:8] == ".godoos." { // 检查文件名是否以 .godoos 开头
log.Printf("Detected .godoos file: %s", event.Name)
fileQueue <- event.Name // 将文件路径放入队列
} else {
if baseName[:1] == "." {
return
}
office.ProcessFile(event.Name)
}
}
case err, ok := <-watcher.Errors:
if !ok {
return
}
log.Printf("Error watching folder %s: %v", folderPath, err)
}
}
}()
err = watcher.Add(folderPath)
if err != nil {
log.Fatalf("Failed to add folder %s to watcher: %v", folderPath, err)
}
// 关闭文件队列
go func() {
<-done
close(fileQueue)
wg.Wait()
}()
<-done
}
func handleGodoosFile(filePath string) {
// 在这里添加对 .godoos 文件的具体处理逻辑
log.Printf("Handling .godoos file: %s", filePath)
}

1
godo/ai/vector/model/vectorlist.go

@ -6,7 +6,6 @@ type VectorList struct {
gorm.Model
Name string `json:"name"`
FilePath string `json:"file_path"`
DbPath string `json:"db_path"`
Engine string `json:"engine"`
EmbeddingUrl string `json:"embedding_url"`
EmbeddingApiKey string `json:"embedding_api_key"`

72
godo/ai/vector/vector.go

@ -1,11 +1,9 @@
package vector
import (
"database/sql"
"fmt"
"godo/ai/vector/model"
"godo/libs"
"os"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/gormlite"
@ -13,7 +11,6 @@ import (
)
var vectorListDb *gorm.DB
var dbPathToSqlDB = make(map[string]*sql.DB)
func init() {
dbPath := libs.GetVectorDb()
@ -26,9 +23,10 @@ func init() {
// CreateVector 创建一个新的 VectorList 记录
func CreateVector(name string, filePath string) (*gorm.DB, error) {
dbPath := libs.GetVectorPath(name)
var list model.VectorList
if !libs.PathExists(filePath) {
return nil, fmt.Errorf("file path not exists")
}
// 检查是否已经存在同名的 VectorList
result := vectorListDb.Where("name = ?", name).First(&list)
if result.Error == nil {
@ -39,7 +37,6 @@ func CreateVector(name string, filePath string) (*gorm.DB, error) {
newList := model.VectorList{
Name: name,
FilePath: filePath, // 根据实际情况设置文件路径
DbPath: dbPath,
}
result = vectorListDb.Create(&newList)
@ -60,70 +57,20 @@ func DeleteVector(name string) error {
return fmt.Errorf("vector list not found")
}
// 关闭数据库连接
err := CloseVectorDb(name)
if err != nil {
return fmt.Errorf("failed to close vector database: %w", err)
}
// 删除数据库文件
dbPath := libs.GetVectorPath(name)
if libs.PathExists(dbPath) {
err := os.Remove(dbPath)
if err != nil {
return fmt.Errorf("failed to delete database file: %w", err)
}
}
return nil
}
// CloseVectorDb 关闭指定名称的 Vector 数据库连接
func CloseVectorDb(name string) error {
dbPath := libs.GetVectorPath(name)
if !libs.PathExists(dbPath) {
return nil
}
sqlDB, exists := dbPathToSqlDB[dbPath]
if !exists {
return fmt.Errorf("no database connection found for path: %s", dbPath)
}
err := sqlDB.Close()
if err != nil {
return fmt.Errorf("failed to close database connection: %w", err)
}
delete(dbPathToSqlDB, name)
return sqlDB.Close()
}
// RenameVectorDb 更改指定名称的 VectorList 的数据库名称
func RenameVectorDb(oldName string, newName string) error {
// 1. 检查并关闭旧的数据库连接(如果已打开)
err := CloseVectorDb(oldName)
if err != nil {
return fmt.Errorf("failed to close vector database: %w", err)
}
// 2. 获取旧的 VectorList 记录
var oldList model.VectorList
result := vectorListDb.Where("name = ?", oldName).First(&oldList)
if result.Error != nil {
return fmt.Errorf("failed to find old vector list: %w", result.Error)
}
//3. 删除旧的数据库文件
oldDbPath := libs.GetVectorPath(oldName)
// 4. 构建新的 DbPath
newDbPath := libs.GetVectorPath(newName)
if libs.PathExists(oldDbPath) {
err := os.Rename(oldDbPath, newDbPath)
if err != nil {
return fmt.Errorf("failed to move old database file to new location: %w", err)
}
}
// 5. 更新 VectorList 记录中的 DbPath 和 Name
oldList.Name = newName
oldList.DbPath = newDbPath
result = vectorListDb.Save(&oldList)
if result.Error != nil {
return fmt.Errorf("failed to update vector list: %w", result.Error)
@ -132,19 +79,6 @@ func RenameVectorDb(oldName string, newName string) error {
return nil
}
// UpdateVector 更新指定名称的 VectorList 记录
func UpdateVector(name string, updates map[string]interface{}) error {
result := vectorListDb.Model(&model.VectorList{}).Where("name = ?", name).Updates(updates)
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return fmt.Errorf("vector list not found")
}
return nil
}
func GetVectorList() []model.VectorList {
var vectorList []model.VectorList
vectorListDb.Find(&vectorList)

48
godo/files/pwd.go

@ -18,6 +18,12 @@ func HandleReadFile(w http.ResponseWriter, r *http.Request) {
libs.ErrorMsg(w, "文件路径不能为空")
return
}
stream := r.URL.Query().Get("stream")
isStream := false
if stream != "" {
isStream = true
}
basePath, err := libs.GetOsDir()
if err != nil {
libs.ErrorMsg(w, err.Error())
@ -45,15 +51,22 @@ func HandleReadFile(w http.ResponseWriter, r *http.Request) {
if !isPwd {
// 未加密文件,直接返回
if len(fileData) > 0 {
text = base64.StdEncoding.EncodeToString(fileData)
if isStream {
w.Header().Set("Content-Type", "application/octet-stream")
_, err := w.Write(fileData)
if err != nil {
libs.ErrorMsg(w, err.Error())
return
}
} else {
if len(fileData) > 0 {
text = base64.StdEncoding.EncodeToString(fileData)
}
//text = base64.StdEncoding.EncodeToString(fileData)
libs.SuccessMsg(w, text, "文件读取成功")
}
// if len(text) > 7 && len(text)%8 == 0 {
// text += " "
// }
//log.Printf("fileData: %s", content)
libs.SuccessMsg(w, text, "文件读取成功")
return
}
filePwd := r.Header.Get("pwd")
@ -80,17 +93,20 @@ func HandleReadFile(w http.ResponseWriter, r *http.Request) {
return
}
}
// if len(decodeFile)%8 == 0 {
// decodeFile = decodeFile + " "
// }
if len(decodeFile) > 0 {
decodeFile = base64.StdEncoding.EncodeToString([]byte(decodeFile))
if isStream {
w.Header().Set("Content-Type", "application/octet-stream")
_, err := w.Write([]byte(decodeFile))
if err != nil {
libs.ErrorMsg(w, err.Error())
return
}
} else {
if len(decodeFile) > 0 {
decodeFile = base64.StdEncoding.EncodeToString([]byte(decodeFile))
}
libs.SuccessMsg(w, decodeFile, "加密文件读取成功")
}
// if len(decodeFile) > 7 && len(decodeFile)%8 == 0 {
// decodeFile += " "
// }
libs.SuccessMsg(w, decodeFile, "加密文件读取成功")
}
func HandleWriteFile(w http.ResponseWriter, r *http.Request) {
path := r.URL.Query().Get("path")

250
godo/office/document.go

@ -0,0 +1,250 @@
/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/
package office
import (
"archive/zip"
"bufio"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"godo/libs"
"os"
"path"
"path/filepath"
"sync"
)
type DocResult struct {
filePath string
newFilePath string
err error
}
func SetDocument(dirPath string) error {
if !libs.PathExists(dirPath) {
return nil
}
var wg sync.WaitGroup
results := make(chan DocResult, 100) // 缓冲通道
err := filepath.Walk(dirPath, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
// 获取文件名
fileName := filepath.Base(filePath)
// 检查文件名是否以点开头
if len(fileName) > 0 && fileName[0] == '.' {
return nil // 跳过以点开头的文件
}
// 获取文件扩展名
ext := filepath.Ext(filePath)
// 检查文件扩展名是否为 .exe
if ext == ".exe" {
return nil // 跳过 .exe 文件
}
wg.Add(1)
go func(filePath string) {
defer wg.Done()
result := ProcessFile(filePath)
results <- result
}(filePath)
}
return nil
})
if err != nil {
return err
}
go func() {
wg.Wait()
close(results)
}()
for result := range results {
if result.err != nil {
fmt.Printf("Failed to process file %s: %v\n", result.filePath, result.err)
} else {
fmt.Printf("Processed file %s and saved JSON to %s\n", result.filePath, result.newFilePath)
}
}
return nil
}
func ProcessFile(filePath string) DocResult {
doc, err := GetDocument(filePath)
if err != nil {
return DocResult{filePath: filePath, err: err}
}
jsonData, err := json.MarshalIndent(doc, "", " ")
if err != nil {
return DocResult{filePath: filePath, err: err}
}
newFileName := ".godoos." + filepath.Base(filePath)
newFilePath := filepath.Join(filepath.Dir(filePath), newFileName)
err = os.WriteFile(newFilePath, jsonData, 0644)
if err != nil {
return DocResult{filePath: filePath, err: err}
}
return DocResult{filePath: filePath, newFilePath: newFilePath, err: nil}
}
func GetDocument(pathname string) (*Document, error) {
if !libs.PathExists(pathname) {
return nil, fmt.Errorf("file does not exist: %s", pathname)
}
abPath, err := filepath.Abs(pathname)
if err != nil {
return nil, err
}
filename := path.Base(pathname)
data := Document{path: pathname, RePath: abPath, Title: filename}
extension := path.Ext(pathname)
_, err = getFileInfoData(&data)
if err != nil {
return &data, err
}
switch extension {
case ".docx":
_, e := getMetaData(&data)
if e != nil {
fmt.Printf("⚠️ %s", e.Error())
}
_, err = getContentData(&data, docx2txt)
case ".pptx":
_, e := getMetaData(&data)
if e != nil {
fmt.Printf("⚠️ %s", e.Error())
}
_, err = getContentData(&data, pptx2txt)
case ".xlsx":
_, e := getMetaData(&data)
if e != nil {
fmt.Printf("⚠️ %s", e.Error())
}
_, err = getContentData(&data, xlsx2txt)
case ".pdf":
_, err = getContentData(&data, pdf2txt)
case ".doc":
_, err = getContentData(&data, doc2txt)
case ".ppt":
_, err = getContentData(&data, ppt2txt)
case ".xls":
_, err = getContentData(&data, xls2txt)
case ".epub":
_, err = getContentData(&data, epub2txt)
case ".odt":
_, err = getContentData(&data, odt2txt)
case ".xml":
_, err = getContentData(&data, xml2txt)
case ".rtf":
_, err = getContentData(&data, rtf2txt)
case ".md":
_, err = getContentData(&data, md2txt)
case ".txt":
_, err = getContentData(&data, text2txt)
case ".xhtml", ".html", ".htm":
_, err = getContentData(&data, html2txt)
case ".json":
_, err = getContentData(&data, json2txt)
}
if err != nil {
return &data, err
}
return &data, nil
}
// Read the meta data of office files (only *.docx, *.xlsx, *.pptx) and insert into the interface
func getMetaData(data *Document) (bool, error) {
file, err := os.Open(data.path)
if err != nil {
return false, err
}
defer file.Close()
meta, err := GetContent(file)
if err != nil {
return false, errors.New("failed to get office meta data")
}
if meta.Title != "" {
data.Title = meta.Title
}
data.Subject = meta.Subject
data.Creator = meta.Creator
data.Keywords = meta.Keywords
data.Description = meta.Description
data.Lastmodifiedby = meta.LastModifiedBy
data.Revision = meta.Revision
data.Category = meta.Category
data.Content = meta.Category
return true, nil
}
func GetContent(document *os.File) (fields XMLContent, err error) {
// Attempt to read the document file directly as a zip file.
z, err := zip.OpenReader(document.Name())
if err != nil {
return fields, errors.New("failed to open the file as zip")
}
defer z.Close()
var xmlFile string
for _, file := range z.File {
if file.Name == "docProps/core.xml" {
rc, err := file.Open()
if err != nil {
return fields, errors.New("failed to open docProps/core.xml")
}
defer rc.Close()
scanner := bufio.NewScanner(rc)
for scanner.Scan() {
xmlFile += scanner.Text()
}
if err := scanner.Err(); err != nil {
return fields, errors.New("failed to read from docProps/core.xml")
}
break // Exit loop after finding and reading core.xml
}
}
// Unmarshal the collected XML content into the XMLContent struct
if err := xml.Unmarshal([]byte(xmlFile), &fields); err != nil {
return fields, errors.New("failed to Unmarshal")
}
return fields, nil
}
// Read the content of office files and insert into the interface
func getContentData(data *Document, reader DocReader) (bool, error) {
content, err := reader(data.path)
if err != nil {
return false, err
}
data.Content = content
data.Split = SplitText(content, 256)
return true, nil
}

244
godo/office/splittext.go

@ -0,0 +1,244 @@
package office
import (
"regexp"
"strings"
)
var minChunkSize = 100
func SplitText(text string, ChunkSize int) []string {
splits := make([]string, 0)
Texts := splitText(text, ChunkSize)
splits = append(splits, Texts...)
return splits
}
// SplitText 是一个将给定文本根据指定的最大长度分割成多个字符串片段的函数。
// text 是需要分割的原始文本。
// maxLength 是可选参数,指定每个分割片段的最大长度。如果未提供,将使用默认值 50。
// 返回值是分割后的字符串片段数组。
func splitText(text string, maxLength ...int) []string {
defaultMaxLength := 256 // 默认的最大长度值为 256
// 检查是否提供了 maxLength 参数,若未提供,则使用默认值
if len(maxLength) == 0 {
maxLength = append(maxLength, defaultMaxLength)
}
// 调用内部函数进行实际的文本分割操作,传入指定的最大长度值
return splitTextInternal(text, maxLength[0])
}
// splitTextInternal 将给定的文本根据指定的最大长度拆分成多个字符串。
// 文本会被处理,以便在拆分时尽可能保持句子的完整性和自然性。
//
// 参数:
//
// text string - 需要拆分的原始文本。
// maxLength int - 每个拆分后字符串的最大长度。
//
// 返回值:
//
// []string - 拆分后的字符串数组。
func splitTextInternal(text string, maxLength int) []string {
// 处理文本,替换多个换行符,压缩空格,并移除多余的换行符
if strings.Contains(text, "\n") {
text = regexp.MustCompile(`\n{3,}`).ReplaceAllString(text, "\n")
text = regexp.MustCompile(`\s`).ReplaceAllString(text, " ")
text = strings.ReplaceAll(text, "\n\n", "")
}
// 为标点符号添加换行符,以改善文本的拆分效果
text = addNewlinesForPunctuation(text)
text = addNewlinesForEllipsis(text)
text = addNewlinesForQuestionMarksAndPeriods(text)
text = strings.TrimSuffix(text, "\n")
// 将处理后的文本按空格拆分成句子
sentences := strings.Fields(text)
// 用于存储最终的句子数组
finalSentences := make([]string, 0)
for i, s := range sentences {
// 如果句子长度超过最大长度,则进行进一步的拆分
if len(s) > maxLength {
// 首先按标点符号拆分句子
punctuatedSentences := splitByPunctuation(s)
for _, p := range punctuatedSentences {
// 如果拆分后的部分仍然超过最大长度,则按多个空格拆分
if len(p) > maxLength {
parts := splitByMultipleSpaces(p)
// 对于每个仍超过最大长度的部分,进一步按引号拆分
for _, part := range parts {
if len(part) > maxLength {
quotedParts := splitByQuotes(part)
// 将拆分得到的部分插入到原始句子列表的适当位置
sentences = appendBefore(i, quotedParts, sentences)
break
}
}
// 将按标点符号拆分的部分插入到原始句子列表的适当位置
sentences = appendBefore(i, punctuatedSentences, sentences)
}
}
} else {
// 如果句子长度小于10个字符,尝试与前一个句子合并
if len(s) < minChunkSize {
// 检查前一个句子,如果它们的长度之和小于 maxLength,则合并它们
if i > 0 && len(finalSentences) > 0 {
prevSentence := finalSentences[len(finalSentences)-1]
if len(prevSentence)+len(s) <= maxLength {
finalSentences[len(finalSentences)-1] += " " + s
continue
}
}
// 如果无法合并,直接添加到结果中
finalSentences = append(finalSentences, s)
}
}
}
return finalSentences
}
// addNewlinesForPunctuation 在文本的标点符号后添加换行符。
// 该函数特别针对中文文本设计,适用于分隔句子以改善可读性。
// 参数:
//
// text string - 需要处理的原始文本。
//
// 返回值:
//
// string - 经过处理,标点符号后添加了换行符的文本。
func addNewlinesForPunctuation(text string) string {
// 使用正则表达式匹配句尾标点符号后紧接着的非句尾标点字符,并在标点符号后添加换行符。
return regexp.MustCompile(`([;;.!?。!?\?])([^”’])`).ReplaceAllString(text, "$1\n$2")
}
// addNewlinesForEllipsis 是一个函数,用于在文本中的省略号后添加换行符。
// 这个函数特别适用于处理文本,以改善其可读性。
// 参数:
//
// text string - 需要处理的原始文本。
//
// 返回值:
//
// string - 经过处理,省略号后添加了换行符的文本。
func addNewlinesForEllipsis(text string) string {
// 使用正则表达式匹配文本中的省略号,并在之后添加换行符。
return regexp.MustCompile(`(\.{6})([^"’”」』])`).ReplaceAllString(text, "$1\n$2")
}
// addNewlinesForQuestionMarksAndPeriods 为文本中的问号和句号添加换行符。
// 该函数使用正则表达式寻找以问号、句号或其他标点符号结尾的句子,并在这些句子后添加换行符,
// 使得每个句子都独占一行。这有助于提高文本的可读性。
// 参数:
//
// text string - 需要处理的原始文本。
//
// 返回值:
//
// string - 经过处理,问号和句号后添加了换行符的文本。
func addNewlinesForQuestionMarksAndPeriods(text string) string {
// 使用正则表达式匹配以特定标点符号结尾的句子,并在这些句子后添加换行符。
return regexp.MustCompile(`([;;!?。!?\?]["’”」』]{0,2})([^;;!?,。!?\?])`).ReplaceAllString(text, "$1\n$2")
}
// splitByPunctuation 函数根据标点符号将句子分割成多个部分。
// 参数:
//
// sentence - 需要分割的原始句子字符串。
//
// 返回值:
//
// []string - 分割后的字符串数组。
func splitByPunctuation(sentence string) []string {
// 使用正则表达式匹配并替换句子中的特定标点符号,以换行符分隔符号和其后的文字。
return strings.Fields(regexp.MustCompile(`([,,.]["’”」』]{0,2})([^,,.])`).ReplaceAllString(sentence, "$1\n$2"))
}
// splitByMultipleSpaces 函数根据多个连续空格或换行符分割输入字符串,并返回一个字符串切片。
// 参数:
//
// part - 需要分割的原始字符串。
//
// 返回值:
//
// 分割后的字符串切片。
func splitByMultipleSpaces(part string) []string {
// 使用正则表达式匹配并替换多个空格或换行符,同时保留这些分隔符与非空字符之间的边界。
return strings.Fields(regexp.MustCompile(`([\n]{1,}| {2,}["’”」』]{0,2})([^\s])`).ReplaceAllString(part, "$1\n$2"))
}
// splitByQuotes 根据引号分割字符串。
// 该函数使用正则表达式寻找被引号包围的单词,并将这些单词与其它内容分割成多个字符串。
//
// 参数:
//
// part string - 需要分割的原始字符串。
//
// 返回值:
//
// []string - 分割后的字符串数组。
func splitByQuotes(part string) []string {
// 使用正则表达式替换引号包围的单词,为其前后添加换行符,然后以换行符为分隔符分割字符串
return strings.Fields(regexp.MustCompile(`( ["’”」』]{0,2})([^ ])`).ReplaceAllString(part, "$1\n$2"))
}
// appendBefore 在指定索引处插入新的句子数组,并返回更新后的句子数组。
// index: 插入位置的索引。
// newSentences: 要插入的新句子数组。
// sentences: 原始句子数组。
// 返回值: 更新后的句子数组。
func appendBefore(index int, newSentences []string, sentences []string) []string {
// 将原数组分为两部分:index之前的部分和index及之后的部分。
// 然后将新句子数组插入到index之前的部分之后,最后合并所有部分。
return append(sentences[:index], append(newSentences, sentences[index:]...)...)
}
// SplitText2 根据最大块大小和指定的分隔符分割纯文本文件内容
func SplitText2(content string, maxChunkSize int, splitChars ...rune) []string {
defaultSplitChars := []rune{',', '.', '\n', '!', '。', ';'}
var chunks []string
currentChunk := ""
if len(splitChars) == 0 {
splitChars = defaultSplitChars
}
// 按照最大块大小和分隔符分割文本
for _, char := range content {
if len(currentChunk)+1 > maxChunkSize && (char == ',' || char == '.') {
chunks = append(chunks, currentChunk)
currentChunk = ""
} else {
currentChunk += string(char)
}
// 如果字符是分隔符,不管长度,都创建一个新的块
if contains(splitChars, char) {
chunks = append(chunks, currentChunk)
currentChunk = ""
}
}
// 添加最后一个块
if currentChunk != "" {
chunks = append(chunks, currentChunk)
}
return chunks
}
// contains 检查 rune 切片是否包含指定的字符
func contains(rs []rune, r rune) bool {
for _, rr := range rs {
if rr == r {
return true
}
}
return false
}

171
godo/office/office.go → godo/office/totxt.go

@ -1,180 +1,16 @@
/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/
package office
import (
"archive/zip"
"bufio"
"bytes"
"encoding/xml"
"errors"
"fmt"
pdf "godo/office/pdf"
xlsx "godo/office/xlsx"
"html"
"os"
"path"
"path/filepath"
"regexp"
"strings"
)
func GetDocument(pathname string) (*Document, error) {
abPath, err := filepath.Abs(pathname)
if err != nil {
return nil, err
}
filename := path.Base(pathname)
data := Document{path: pathname, RePath: abPath, Title: filename}
extension := path.Ext(pathname)
_, err = getFileInfoData(&data)
if err != nil {
return &data, err
}
switch extension {
case ".docx":
_, e := getMetaData(&data)
if e != nil {
fmt.Printf("⚠️ %s", e.Error())
}
_, err = getContentData(&data, docx2txt)
case ".pptx":
_, e := getMetaData(&data)
if e != nil {
fmt.Printf("⚠️ %s", e.Error())
}
_, err = getContentData(&data, pptx2txt)
case ".xlsx":
_, e := getMetaData(&data)
if e != nil {
fmt.Printf("⚠️ %s", e.Error())
}
_, err = getContentData(&data, xlsx2txt)
case ".pdf":
_, err = getContentData(&data, pdf2txt)
case ".doc":
_, err = getContentData(&data, doc2txt)
case ".ppt":
_, err = getContentData(&data, ppt2txt)
case ".xls":
_, err = getContentData(&data, xls2txt)
case ".epub":
_, err = getContentData(&data, epub2txt)
case ".odt":
_, err = getContentData(&data, odt2txt)
case ".xml":
_, err = getContentData(&data, xml2txt)
case ".rtf":
_, err = getContentData(&data, rtf2txt)
case ".md":
_, err = getContentData(&data, md2txt)
case ".txt":
_, err = getContentData(&data, text2txt)
case ".xhtml", ".html", ".htm":
_, err = getContentData(&data, html2txt)
case ".json":
_, err = getContentData(&data, json2txt)
}
if err != nil {
return &data, err
}
return &data, nil
}
// Read the meta data of office files (only *.docx, *.xlsx, *.pptx) and insert into the interface
func getMetaData(data *Document) (bool, error) {
file, err := os.Open(data.path)
if err != nil {
return false, err
}
defer file.Close()
meta, err := GetContent(file)
if err != nil {
return false, errors.New("failed to get office meta data")
}
if meta.Title != "" {
data.Title = meta.Title
}
data.Subject = meta.Subject
data.Creator = meta.Creator
data.Keywords = meta.Keywords
data.Description = meta.Description
data.Lastmodifiedby = meta.LastModifiedBy
data.Revision = meta.Revision
data.Category = meta.Category
data.Content = meta.Category
return true, nil
}
func GetContent(document *os.File) (fields XMLContent, err error) {
// Attempt to read the document file directly as a zip file.
z, err := zip.OpenReader(document.Name())
if err != nil {
return fields, errors.New("failed to open the file as zip")
}
defer z.Close()
var xmlFile string
for _, file := range z.File {
if file.Name == "docProps/core.xml" {
rc, err := file.Open()
if err != nil {
return fields, errors.New("failed to open docProps/core.xml")
}
defer rc.Close()
scanner := bufio.NewScanner(rc)
for scanner.Scan() {
xmlFile += scanner.Text()
}
if err := scanner.Err(); err != nil {
return fields, errors.New("failed to read from docProps/core.xml")
}
break // Exit loop after finding and reading core.xml
}
}
// Unmarshal the collected XML content into the XMLContent struct
if err := xml.Unmarshal([]byte(xmlFile), &fields); err != nil {
return fields, errors.New("failed to Unmarshal")
}
return fields, nil
}
// Read the content of office files and insert into the interface
func getContentData(data *Document, reader DocReader) (bool, error) {
content, err := reader(data.path)
if err != nil {
return false, err
}
data.Content = content
return true, nil
}
// Read the file information of any files and insert into the interface
func removeStrangeChars(input string) string {
// Define the regex pattern for allowed characters
re := regexp.MustCompile("[�\x13\x0b]+")
// Replace all disallowed characters with an empty string
return re.ReplaceAllString(input, " ")
}
func docx2txt(filename string) (string, error) {
data_docx, err := ReadDocxFile(filename) // Read data from docx file
if err != nil {
@ -319,3 +155,10 @@ func xls2txt(filename string) (string, error) {
// fmt.Println(text_xls)
return text_xls, nil
}
func removeStrangeChars(input string) string {
// Define the regex pattern for allowed characters
re := regexp.MustCompile("[�\x13\x0b]+")
// Replace all disallowed characters with an empty string
return re.ReplaceAllString(input, " ")
}

1
godo/office/types.go

@ -25,6 +25,7 @@ type Document struct {
Revision string `json:"revision"`
Category string `json:"category"`
Content string `json:"content"`
Split []string `json:"split"`
Modifytime time.Time `json:"modified"`
Createtime time.Time `json:"created"`
Accesstime time.Time `json:"accessed"`

29
packages/pptx/README.md

@ -1 +1,28 @@
### fork from https://github.com/pipipi-pikachu/PPTist/
### fork from https://github.com/pipipi-pikachu/PPTist/
## 修改说明
- 2024-08-18 : 新增三套模板,修改保存文件方式
## 模板类型 tplType
- home 首页
- contents 一级目录列表
- catePage 一级目录首页
- cateItem 一级目录内容,包含适配内容num
- contentPage 二级目录首页
- contentItem 二级目录内容,包含适配内容num
- end 尾页
# 标题 title
- 副标题 subTitle
## 一级目录1 catalog
- 二级目录说明1
### 二级目录11
- 三级目录说明1
- 三级目录说明2
### 二级目录12
## 一级目录12
- 二级目录说明2
### 二级目录21
### 二级目录22

19
packages/pptx/package-lock.json

@ -4884,7 +4884,7 @@
"version": "5.3.3",
"resolved": "https://registry.npmmirror.com/typescript/-/typescript-5.3.3.tgz",
"integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
"dev": true,
"devOptional": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@ -5819,7 +5819,8 @@
"@icon-park/vue-next": {
"version": "1.4.2",
"resolved": "https://registry.npmmirror.com/@icon-park/vue-next/-/vue-next-1.4.2.tgz",
"integrity": "sha512-+QklF255wkfBOabY+xw6FAI0Bwln/RhdwCunNy/9sKdKuChtaU67QZqU67KGAvZUTeeBgsL+yaHHxqfQeGZXEQ=="
"integrity": "sha512-+QklF255wkfBOabY+xw6FAI0Bwln/RhdwCunNy/9sKdKuChtaU67QZqU67KGAvZUTeeBgsL+yaHHxqfQeGZXEQ==",
"requires": {}
},
"@jridgewell/sourcemap-codec": {
"version": "1.4.15",
@ -6133,7 +6134,8 @@
"version": "4.6.2",
"resolved": "https://registry.npmmirror.com/@vitejs/plugin-vue/-/plugin-vue-4.6.2.tgz",
"integrity": "sha512-kqf7SGFoG+80aZG6Pf+gsZIVvGSCKE98JbiWqcCV9cThtg91Jav0yvYFC9Zb+jKetNGF6ZKeoaxgZfND21fWKw==",
"dev": true
"dev": true,
"requires": {}
},
"@volar/language-core": {
"version": "1.11.1",
@ -6299,7 +6301,8 @@
"version": "5.3.2",
"resolved": "https://registry.npmmirror.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
"integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
"dev": true
"dev": true,
"requires": {}
},
"ajv": {
"version": "6.12.6",
@ -8062,7 +8065,8 @@
"vue-demi": {
"version": "0.14.6",
"resolved": "https://registry.npmmirror.com/vue-demi/-/vue-demi-0.14.6.tgz",
"integrity": "sha512-8QA7wrYSHKaYgUxDA5ZC24w+eHm3sYCbp0EzcDwKqN3p6HqtTCGR/GVsPyZW92unff4UlcSh++lmqDWN3ZIq4w=="
"integrity": "sha512-8QA7wrYSHKaYgUxDA5ZC24w+eHm3sYCbp0EzcDwKqN3p6HqtTCGR/GVsPyZW92unff4UlcSh++lmqDWN3ZIq4w==",
"requires": {}
}
}
},
@ -8796,7 +8800,8 @@
"version": "1.0.3",
"resolved": "https://registry.npmmirror.com/ts-api-utils/-/ts-api-utils-1.0.3.tgz",
"integrity": "sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==",
"dev": true
"dev": true,
"requires": {}
},
"txml": {
"version": "5.1.1",
@ -8825,7 +8830,7 @@
"version": "5.3.3",
"resolved": "https://registry.npmmirror.com/typescript/-/typescript-5.3.3.tgz",
"integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
"dev": true
"devOptional": true
},
"undici-types": {
"version": "5.26.5",

3
packages/pptx/package.json

@ -9,8 +9,7 @@
"preview": "vite preview",
"build-only": "vite build",
"type-check": "vue-tsc --build --force",
"lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore",
"prepare": "husky install"
"lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore"
},
"dependencies": {
"@icon-park/vue-next": "^1.4.2",

Loading…
Cancel
Save