mirror of https://gitee.com/godoos/godoos.git
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
321 lines
9.1 KiB
321 lines
9.1 KiB
/*
|
|
Licensed to the Apache Software Foundation (ASF) under one
|
|
or more contributor license agreements. See the NOTICE file
|
|
distributed with this work for additional information
|
|
regarding copyright ownership. The ASF licenses this file
|
|
to you under the Apache License, Version 2.0 (the
|
|
"License"); you may not use this file except in compliance
|
|
with the License. You may obtain a copy of the License at
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
Unless required by applicable law or agreed to in writing,
|
|
software distributed under the License is distributed on an
|
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
KIND, either express or implied. See the License for the
|
|
specific language governing permissions and limitations
|
|
under the License.
|
|
*/
|
|
|
|
package office
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bufio"
|
|
"bytes"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
pdf "godo/office/pdf"
|
|
xlsx "godo/office/xlsx"
|
|
"html"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
func GetDocument(pathname string) (*Document, error) {
|
|
abPath, err := filepath.Abs(pathname)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
filename := path.Base(pathname)
|
|
data := Document{path: pathname, RePath: abPath, Title: filename}
|
|
extension := path.Ext(pathname)
|
|
_, err = getFileInfoData(&data)
|
|
if err != nil {
|
|
return &data, err
|
|
}
|
|
switch extension {
|
|
case ".docx":
|
|
_, e := getMetaData(&data)
|
|
if e != nil {
|
|
fmt.Printf("⚠️ %s", e.Error())
|
|
}
|
|
_, err = getContentData(&data, docx2txt)
|
|
case ".pptx":
|
|
_, e := getMetaData(&data)
|
|
if e != nil {
|
|
fmt.Printf("⚠️ %s", e.Error())
|
|
}
|
|
_, err = getContentData(&data, pptx2txt)
|
|
case ".xlsx":
|
|
_, e := getMetaData(&data)
|
|
if e != nil {
|
|
fmt.Printf("⚠️ %s", e.Error())
|
|
}
|
|
_, err = getContentData(&data, xlsx2txt)
|
|
case ".pdf":
|
|
_, err = getContentData(&data, pdf2txt)
|
|
case ".doc":
|
|
_, err = getContentData(&data, doc2txt)
|
|
case ".ppt":
|
|
_, err = getContentData(&data, ppt2txt)
|
|
case ".xls":
|
|
_, err = getContentData(&data, xls2txt)
|
|
case ".epub":
|
|
_, err = getContentData(&data, epub2txt)
|
|
case ".odt":
|
|
_, err = getContentData(&data, odt2txt)
|
|
case ".xml":
|
|
_, err = getContentData(&data, xml2txt)
|
|
case ".rtf":
|
|
_, err = getContentData(&data, rtf2txt)
|
|
case ".md":
|
|
_, err = getContentData(&data, md2txt)
|
|
case ".txt":
|
|
_, err = getContentData(&data, text2txt)
|
|
case ".xhtml", ".html", ".htm":
|
|
_, err = getContentData(&data, html2txt)
|
|
case ".json":
|
|
_, err = getContentData(&data, json2txt)
|
|
}
|
|
if err != nil {
|
|
return &data, err
|
|
}
|
|
return &data, nil
|
|
}
|
|
|
|
// Read the meta data of office files (only *.docx, *.xlsx, *.pptx) and insert into the interface
|
|
func getMetaData(data *Document) (bool, error) {
|
|
file, err := os.Open(data.path)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
defer file.Close()
|
|
meta, err := GetContent(file)
|
|
if err != nil {
|
|
return false, errors.New("failed to get office meta data")
|
|
}
|
|
if meta.Title != "" {
|
|
data.Title = meta.Title
|
|
}
|
|
data.Subject = meta.Subject
|
|
data.Creator = meta.Creator
|
|
data.Keywords = meta.Keywords
|
|
data.Description = meta.Description
|
|
data.Lastmodifiedby = meta.LastModifiedBy
|
|
data.Revision = meta.Revision
|
|
data.Category = meta.Category
|
|
data.Content = meta.Category
|
|
return true, nil
|
|
}
|
|
func GetContent(document *os.File) (fields XMLContent, err error) {
|
|
// Attempt to read the document file directly as a zip file.
|
|
z, err := zip.OpenReader(document.Name())
|
|
if err != nil {
|
|
return fields, errors.New("failed to open the file as zip")
|
|
}
|
|
defer z.Close()
|
|
|
|
var xmlFile string
|
|
for _, file := range z.File {
|
|
if file.Name == "docProps/core.xml" {
|
|
rc, err := file.Open()
|
|
if err != nil {
|
|
return fields, errors.New("failed to open docProps/core.xml")
|
|
}
|
|
defer rc.Close()
|
|
|
|
scanner := bufio.NewScanner(rc)
|
|
for scanner.Scan() {
|
|
xmlFile += scanner.Text()
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return fields, errors.New("failed to read from docProps/core.xml")
|
|
}
|
|
break // Exit loop after finding and reading core.xml
|
|
}
|
|
}
|
|
|
|
// Unmarshal the collected XML content into the XMLContent struct
|
|
if err := xml.Unmarshal([]byte(xmlFile), &fields); err != nil {
|
|
return fields, errors.New("failed to Unmarshal")
|
|
}
|
|
|
|
return fields, nil
|
|
}
|
|
|
|
// Read the content of office files and insert into the interface
|
|
func getContentData(data *Document, reader DocReader) (bool, error) {
|
|
content, err := reader(data.path)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
data.Content = content
|
|
return true, nil
|
|
}
|
|
|
|
// Read the file information of any files and insert into the interface
|
|
|
|
func removeStrangeChars(input string) string {
|
|
// Define the regex pattern for allowed characters
|
|
re := regexp.MustCompile("[�\x13\x0b]+")
|
|
// Replace all disallowed characters with an empty string
|
|
return re.ReplaceAllString(input, " ")
|
|
}
|
|
|
|
func docx2txt(filename string) (string, error) {
|
|
data_docx, err := ReadDocxFile(filename) // Read data from docx file
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer data_docx.Close()
|
|
text_docx := data_docx.Editable().GetContent() // Get whole docx data as XML formated text
|
|
text_docx = PARA_RE.ReplaceAllString(text_docx, "\n") // Replace the end of paragraphs (</w:p) with /n
|
|
text_docx = TAG_RE.ReplaceAllString(text_docx, "") // Remove all the tags to extract the content
|
|
text_docx = html.UnescapeString(text_docx) // Replace all the html entities (e.g. &)
|
|
|
|
// fmt.Println(text_docx)
|
|
return text_docx, nil
|
|
}
|
|
|
|
func pptx2txt(filename string) (string, error) {
|
|
data_pptx, err := ReadPowerPoint(filename) // Read data from pptx file
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
data_pptx.DeletePassWord()
|
|
slides_pptx := data_pptx.GetSlidesContent() // Get pptx slides data as an array of XML formated text
|
|
var text_pptx string
|
|
for i := range slides_pptx {
|
|
slide_text_pptx := PARA_RE.ReplaceAllString(slides_pptx[i], "\n") // Replace the end of paragraphs (</w:p) with /n
|
|
slide_text_pptx = TAG_RE.ReplaceAllString(slide_text_pptx, "") // Remove all the tags to extract the content
|
|
slide_text_pptx = html.UnescapeString(slide_text_pptx) // Replace all the html entities (e.g. &)
|
|
if slide_text_pptx != "" { // Save all slides as ONE string
|
|
if text_pptx != "" {
|
|
text_pptx = fmt.Sprintf("%s\n%s", text_pptx, slide_text_pptx)
|
|
} else {
|
|
text_pptx = fmt.Sprintf("%s%s", text_pptx, slide_text_pptx)
|
|
}
|
|
}
|
|
}
|
|
// fmt.Println(text_pptx)
|
|
return text_pptx, nil
|
|
}
|
|
|
|
func xlsx2txt(filename string) (string, error) {
|
|
data_xlsx, err := xlsx.OpenFile(filename) // Read data from xlsx file
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer data_xlsx.Close()
|
|
|
|
var rows_xlsx string
|
|
for _, sheet := range data_xlsx.Sheets { // For each sheet of the file
|
|
for row := range data_xlsx.ReadRows(sheet) { // For each row of the sheet
|
|
text_row := ""
|
|
for i, col := range row.Cells { // Concatenate cells of the row with tab separator
|
|
if i > 0 {
|
|
text_row = fmt.Sprintf("%s\t%s", text_row, col.Value)
|
|
} else {
|
|
text_row = fmt.Sprintf("%s%s", text_row, col.Value)
|
|
}
|
|
}
|
|
if rows_xlsx != "" { // Save all rows as ONE string
|
|
rows_xlsx = fmt.Sprintf("%s\n%s", rows_xlsx, text_row)
|
|
} else {
|
|
rows_xlsx = fmt.Sprintf("%s%s", rows_xlsx, text_row)
|
|
}
|
|
}
|
|
}
|
|
// fmt.Println(rows_xlsx)
|
|
return rows_xlsx, nil
|
|
}
|
|
|
|
func pdf2txt(filename string) (string, error) { // BUG: Cannot get text from specific (or really malformed?) pages
|
|
file_pdf, data_pdf, err := pdf.Open(filename) // Read data from pdf file
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file_pdf.Close()
|
|
|
|
var buff_pdf bytes.Buffer
|
|
bytes_pdf, err := data_pdf.GetPlainText() // Get text of entire pdf file
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
buff_pdf.ReadFrom(bytes_pdf)
|
|
text_pdf := buff_pdf.String()
|
|
// fmt.Println(text_pdf)
|
|
return text_pdf, nil
|
|
}
|
|
|
|
func doc2txt(filename string) (string, error) {
|
|
file_doc, _ := os.Open(filename) // Open doc file
|
|
data_doc, err := DOC2Text(file_doc) // Read data from a doc file
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file_doc.Close()
|
|
|
|
actual := data_doc.(*bytes.Buffer) // Buffer for hold line text of doc file
|
|
text_doc := ""
|
|
for aline, err := actual.ReadString('\r'); err == nil; aline, err = actual.ReadString('\r') { // Get text by line
|
|
aline = strings.Trim(aline, " \n\r")
|
|
if aline != "" {
|
|
if text_doc != "" {
|
|
text_doc = fmt.Sprintf("%s\n%s", text_doc, removeStrangeChars(aline))
|
|
} else {
|
|
text_doc = fmt.Sprintf("%s%s", text_doc, removeStrangeChars(aline))
|
|
}
|
|
}
|
|
}
|
|
text_doc = removeStrangeChars(text_doc)
|
|
// fmt.Println(text_doc)
|
|
return text_doc, nil
|
|
}
|
|
|
|
func ppt2txt(filename string) (string, error) {
|
|
file_ppt, err := os.Open(filename) // Open ppt file
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file_ppt.Close()
|
|
|
|
text_ppt, err := ExtractPPTText(file_ppt) // Read text from a ppt file
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
text_ppt = removeStrangeChars(text_ppt)
|
|
// fmt.Println(text_ppt)
|
|
return text_ppt, nil
|
|
}
|
|
|
|
func xls2txt(filename string) (string, error) {
|
|
file_xls, err := os.Open(filename) // Open xls file
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file_xls.Close()
|
|
|
|
text_xls, err := XLS2Text(file_xls) // Convert xls data to an array of rows (include all sheets)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
text_xls = removeStrangeChars(text_xls)
|
|
// fmt.Println(text_xls)
|
|
return text_xls, nil
|
|
}
|
|
|