From 39cdeba29b177fe8fa76e8fb5aeccac9cec862cb Mon Sep 17 00:00:00 2001 From: Elbahkiry Date: Thu, 8 Aug 2024 15:37:48 +0300 Subject: [PATCH 1/6] deal with files without extension and update ioutil to io --- api/icap-request.go | 9 ++++----- consts/consts.go | 2 +- icap-client/header.go | 10 +++++----- icap-client/request_test.go | 10 +++++----- icap/request.go | 7 +++---- icap/response.go | 5 ++--- icap/response_test.go | 3 +-- .../services-utilities/ContentTypes/contentType.go | 3 +-- .../general-functions/general-functions.go | 12 ++++++++++-- 9 files changed, 32 insertions(+), 29 deletions(-) diff --git a/api/icap-request.go b/api/icap-request.go index cdc5393e..d22b43c7 100644 --- a/api/icap-request.go +++ b/api/icap-request.go @@ -11,7 +11,6 @@ import ( "icapeg/logging" "icapeg/service" "io" - "io/ioutil" "math/rand" "net/http" "strconv" @@ -140,7 +139,7 @@ func (i *ICAPRequest) RequestProcessing(xICAPMetadata string) { } else { i.req.OrgRequest = new } - body, _ := ioutil.ReadAll(i.req.Request.Body) + body, _ := io.ReadAll(i.req.Request.Body) i.req.OrgRequest.Body = io.NopCloser(bytes.NewBuffer(body)) i.req.OrgRequest.Header = i.req.Request.Header i.req.OrgRequest.Header.Set(utils.ContentLength, strconv.Itoa(len(body))) @@ -283,7 +282,7 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { IcapStatusCode = utils.OkStatusCodeStr if i.methodName == utils.ICAPModeReq { IcapStatusCode = utils.OkStatusCodeStr - body, _ := ioutil.ReadAll(i.req.OrgRequest.Body) + body, _ := io.ReadAll(i.req.OrgRequest.Body) i.req.Request.Body = io.NopCloser(bytes.NewBuffer(body)) i.req.Request.Header.Set(utils.ContentLength, strconv.Itoa(len(body))) defer i.req.Request.Body.Close() @@ -438,12 +437,12 @@ func (i *ICAPRequest) shadowService(xICAPMetadata string) { } else { if i.req.Method == "REQMOD" { i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Request, true) - tempBody, _ := ioutil.ReadAll(i.req.Request.Body) + tempBody, _ := io.ReadAll(i.req.Request.Body) i.w.Write(tempBody) i.req.Request.Body = io.NopCloser(bytes.NewBuffer(tempBody)) } else if i.req.Method == "RESPMOD" { i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Response, true) - tempBody, _ := ioutil.ReadAll(i.req.Response.Body) + tempBody, _ := io.ReadAll(i.req.Response.Body) i.w.Write(tempBody) i.req.Response.Body = io.NopCloser(bytes.NewBuffer(tempBody)) } diff --git a/consts/consts.go b/consts/consts.go index 0b8fa9c4..9c51f47e 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -15,7 +15,7 @@ const ( // the common constants const ( - Unknown = "unknown" + Unknown = "!" Any = "*" NoModificationStatusCodeStr = 204 BadRequestStatusCodeStr = 400 diff --git a/icap-client/header.go b/icap-client/header.go index 67b4d2ef..f9cc9df6 100644 --- a/icap-client/header.go +++ b/icap-client/header.go @@ -2,7 +2,7 @@ package icapclient import ( "bytes" - "io/ioutil" + "io" "net/http" "os" "strconv" @@ -23,7 +23,7 @@ func (r *Request) SetPreview(maxBytes int) error { } if r.HTTPRequest.Body != nil { var err error - bodyBytes, err = ioutil.ReadAll(r.HTTPRequest.Body) + bodyBytes, err = io.ReadAll(r.HTTPRequest.Body) if err != nil { return err @@ -40,7 +40,7 @@ func (r *Request) SetPreview(maxBytes int) error { if r.HTTPResponse.Body != nil { var err error - bodyBytes, err = ioutil.ReadAll(r.HTTPResponse.Body) + bodyBytes, err = io.ReadAll(r.HTTPResponse.Body) if err != nil { return err @@ -65,11 +65,11 @@ func (r *Request) SetPreview(maxBytes int) error { // returning the body back to the http message depending on the request method if r.Method == MethodREQMOD { - r.HTTPRequest.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) + r.HTTPRequest.Body = io.NopCloser(bytes.NewReader(bodyBytes)) } if r.Method == MethodRESPMOD { - r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) + r.HTTPResponse.Body = io.NopCloser(bytes.NewReader(bodyBytes)) } // finally assinging the preview informations including setting the header diff --git a/icap-client/request_test.go b/icap-client/request_test.go index 63c2fd07..39dc3aa1 100644 --- a/icap-client/request_test.go +++ b/icap-client/request_test.go @@ -3,7 +3,7 @@ package icapclient import ( "bytes" "errors" - "io/ioutil" + "io" "net/http" "os" "reflect" @@ -198,7 +198,7 @@ func TestRequest(t *testing.T) { "Content-Length": []string{"11"}, }, ContentLength: 11, - Body: ioutil.NopCloser(strings.NewReader("Hello World")), + Body: io.NopCloser(strings.NewReader("Hello World")), } req, _ := NewRequest(MethodRESPMOD, "icap://localhost:1344/something", httpReq, httpResp) @@ -392,7 +392,7 @@ func TestRequest(t *testing.T) { "Content-Length": []string{strconv.Itoa(bodyData.Len())}, }, ContentLength: int64(bodyData.Len()), - Body: ioutil.NopCloser(strings.NewReader(sample.bodyStr)), + Body: io.NopCloser(strings.NewReader(sample.bodyStr)), } req, _ = NewRequest(sample.reqMethod, "icap://localhost:1344/something", httpReq, httpResp) } @@ -409,11 +409,11 @@ func TestRequest(t *testing.T) { var bdyBytes []byte if sample.reqMethod == MethodREQMOD { - bdyBytes, _ = ioutil.ReadAll(req.HTTPRequest.Body) + bdyBytes, _ = io.ReadAll(req.HTTPRequest.Body) } if sample.reqMethod == MethodRESPMOD { - bdyBytes, _ = ioutil.ReadAll(req.HTTPResponse.Body) + bdyBytes, _ = io.ReadAll(req.HTTPResponse.Body) } if string(bdyBytes) != sample.bodyStr { diff --git a/icap/request.go b/icap/request.go index 8426d4ff..865103a1 100644 --- a/icap/request.go +++ b/icap/request.go @@ -12,7 +12,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "net/http" "net/textproto" "net/url" @@ -150,7 +149,7 @@ func ReadRequest(b *bufio.ReadWriter) (req *Request, err error) { if hasBody { if p := req.Header.Get("Preview"); p != "" { - req.Preview, err = ioutil.ReadAll(newChunkedReader(b)) + req.Preview, err = io.ReadAll(newChunkedReader(b)) origBuf = b origReader = bytes.NewBuffer(req.Preview) req.EndIndicator = "0" @@ -164,9 +163,9 @@ func ReadRequest(b *bufio.ReadWriter) (req *Request, err error) { } } var r io.Reader = bytes.NewBuffer(req.Preview) - bodyReader = ioutil.NopCloser(r) + bodyReader = io.NopCloser(r) } else { - bodyReader = ioutil.NopCloser(newChunkedReader(b)) + bodyReader = io.NopCloser(newChunkedReader(b)) } } diff --git a/icap/response.go b/icap/response.go index 13cd1df1..7eb431d9 100644 --- a/icap/response.go +++ b/icap/response.go @@ -11,7 +11,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "net/http" "net/http/httputil" @@ -146,10 +145,10 @@ func (w *respWriter) WriteHeader(code int, httpMessage interface{}, hasBody bool if hasBody { switch msg := httpMessage.(type) { case *http.Response: - requestBody, _ := ioutil.ReadAll(msg.Body) + requestBody, _ := io.ReadAll(msg.Body) w.Write(requestBody) case *http.Request: - requestBody, _ := ioutil.ReadAll(msg.Body) + requestBody, _ := io.ReadAll(msg.Body) w.Write(requestBody) } diff --git a/icap/response_test.go b/icap/response_test.go index 81fdce98..d7fdd3b3 100644 --- a/icap/response_test.go +++ b/icap/response_test.go @@ -6,7 +6,6 @@ package icap import ( "io" - "io/ioutil" ) const serverAddr = "localhost:11344" @@ -76,7 +75,7 @@ func HandleREQMOD2(w ResponseWriter, req *Request) { req.Request.Header.Set("Accept", "text/html, text/plain, image/gif") req.Request.Header.Set("Accept-Encoding", "gzip, compress") - body, _ := ioutil.ReadAll(req.Request.Body) + body, _ := io.ReadAll(req.Request.Body) newBody := string(body) + " ICAP powered!" w.WriteHeader(200, req.Request, true) diff --git a/service/services-utilities/ContentTypes/contentType.go b/service/services-utilities/ContentTypes/contentType.go index c0b698b7..dc553eab 100644 --- a/service/services-utilities/ContentTypes/contentType.go +++ b/service/services-utilities/ContentTypes/contentType.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "io" - "io/ioutil" "net/http" "strings" ) @@ -28,7 +27,7 @@ func GetContentType(req *http.Request) ContentType { //first is that file encoded in base64 and second is that file is actually a JSON file, //so we convert the JSON file to a map var data map[string]interface{} - body, _ := ioutil.ReadAll(req.Body) + body, _ := io.ReadAll(req.Body) _ = json.Unmarshal(body, &data) //checking if there is key equals "Base64", the file would be encoded diff --git a/service/services-utilities/general-functions/general-functions.go b/service/services-utilities/general-functions/general-functions.go index b5f7b36f..fa5f1881 100644 --- a/service/services-utilities/general-functions/general-functions.go +++ b/service/services-utilities/general-functions/general-functions.go @@ -13,10 +13,10 @@ import ( "icapeg/service/services-utilities/ContentTypes" "image" "io" - "io/ioutil" "mime" "net/http" "path" + "path/filepath" "strconv" "strings" @@ -189,7 +189,7 @@ func (f *GeneralFunc) DecompressGzipBody(file *bytes.Buffer) (*bytes.Buffer, err logging.Logger.Info(utils.PrepareLogMsg(f.xICAPMetadata, "decompressing the HTTP message body")) reader, err := gzip.NewReader(file) defer reader.Close() - result, err := ioutil.ReadAll(reader) + result, err := io.ReadAll(reader) if err != nil { return nil, err } @@ -306,6 +306,14 @@ func (f *GeneralFunc) GetFileName(serviceName string, xICAPMetadata string) stri if len(filename) < 2 { return "unnamed_file" } + // Handle cases where the file extension is empty or just a dot + ext := filepath.Ext(filename) + if ext == "" || ext == "." { + filename = filename + "." + utils.Unknown + logging.Logger.Debug(utils.PrepareLogMsg(f.xICAPMetadata, + "File extension was empty or a dot, set to unknown: "+filename)) + } + logging.Logger.Info(utils.PrepareLogMsg(f.xICAPMetadata, serviceName+" file name : "+filename)) return filename From 61fd23c5333b9dddcb1cbb8b6c15ddbada4e0818 Mon Sep 17 00:00:00 2001 From: Elbahkiry Date: Tue, 3 Sep 2024 10:45:30 +0300 Subject: [PATCH 2/6] add storage client and use exeption page instead of block page --- README.md | 18 +- api/icap-request.go | 78 ++-- block-page.html | 1 + config.toml | 2 + config/config.go | 34 +- consts/consts.go | 2 +- go.mod | 1 + go.sum | 1 + http-message/httpMessage.go | 15 +- icap-client/client_test.go | 8 +- icap-client/test_server.go | 14 +- icap/bridge.go | 2 +- icap/mux.go | 8 +- icap/request.go | 52 ++- icap/response.go | 22 +- icap/response_test.go | 2 +- icap/server.go | 5 + server/server.go | 29 +- .../general-functions/general-functions.go | 66 +-- service/services/clamav/clamav.go | 25 +- service/services/clhashlookup/clhashlookup.go | 385 +++++++++-------- service/services/clhashlookup/config.go | 4 + service/services/echo/echo.go | 14 +- service/services/hashlocal/config.go | 4 + service/services/hashlocal/hashlocal.go | 387 ++++++++++-------- storage/auto.go | 298 ++++++++++++++ storage/disk.go | 157 +++++++ storage/memory.go | 90 ++++ storage/storage.go | 17 + temp/exception-page.html | 2 +- 30 files changed, 1279 insertions(+), 464 deletions(-) create mode 100644 storage/auto.go create mode 100644 storage/disk.go create mode 100644 storage/memory.go create mode 100644 storage/storage.go diff --git a/README.md b/README.md index 86499a47..5c72141f 100644 --- a/README.md +++ b/README.md @@ -98,13 +98,13 @@ You should see something like, ```ICAP server is running on localhost:1344 ...`` - **echo** > simply it's like when you bing url,so what we send will be received -- ## clamav** +- ## **clamav** > ClamAV is an open source (GPLv2) anti-virus toolkit, designed especially for e-mail scanning on mail gateways. It provides a number of utilities including a flexible and scalable multi-threaded daemon also helps to scan file quickly. -- ## clhashlookup** -> simply it helps to scan each file we need to check before we send to Api. +- ## **clhashlookup** +> simply it helps to scan each file we need to check before we send to API. -- ## hashlocal** +- ## **hashlocal** > we will pretend that you want to download a file. Still, you don't know if it is safe or not, so using this service helps you calculate the hash value of any file you download through the icapeg, then it checks if this hash value is available or not in hash_file if it is in the hash_file returns back an exception page, so if you tried to downloadany file of eicar test vrius it will appear an exception page,cause the hash file of thosefile is foun in our hash_file. @@ -131,6 +131,8 @@ You should see something like, ```ICAP server is running on localhost:1344 ...`` port = 1344 services= ["echo", "virustotal", "clamav", "cloudmersive"] debugging_headers=true + max_file_size_on_memory = 10 # in megabytes + write_path_on_disk = "./tmp" ``` - **port** @@ -153,7 +155,13 @@ You should see something like, ```ICAP server is running on localhost:1344 ...`` - **false**: Debugging headers should not be displayed with ICAP headers. - Any port number that isn't used in your machine. + - **max_file_size_on_memory** (MB) + The maximum file size in megabytes that will be handled in memory. Files exceeding this size will be stored and processed on disk. The default value is `10` MB. + + - **write_path_on_disk** + + The directory path where files exceeding `max_file_size_on_memory` will be stored on disk. The default path is `"./tmp"`. ## echo - **[echo] section** @@ -407,7 +415,7 @@ You should see something like, ```ICAP server is running on localhost:1344 ...`` http_exception_has_body = true exception_page = "./temp/exception-page.html" # Location of the exception page for this service ``` - - ### **New used variables ** + - ### **New used variables** - **socket_path** It is a string variable it helps sending the HTTP msg body to the ClamAV through antivirus socket. diff --git a/api/icap-request.go b/api/icap-request.go index d22b43c7..72ab6037 100644 --- a/api/icap-request.go +++ b/api/icap-request.go @@ -11,6 +11,7 @@ import ( "icapeg/logging" "icapeg/service" "io" + "log" "math/rand" "net/http" "strconv" @@ -62,7 +63,7 @@ func (i *ICAPRequest) RequestInitialization() (string, error) { logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "checking if the service doesn't exist in toml file")) i.serviceName = i.req.URL.Path[1:len(i.req.URL.Path)] if !i.isServiceExists(xICAPMetadata) { - i.w.WriteHeader(utils.ICAPServiceNotFoundCodeStr, nil, false) + i.w.WriteHeader(utils.ICAPServiceNotFoundCodeStr, nil, false, i.req) err := errors.New("service doesn't exist") logging.Logger.Error(err.Error()) return xICAPMetadata, err @@ -73,7 +74,7 @@ func (i *ICAPRequest) RequestInitialization() (string, error) { i.methodName = i.req.Method if i.methodName != "options" { if !i.isMethodAllowed(xICAPMetadata) { - i.w.WriteHeader(utils.MethodNotAllowedForServiceCodeStr, nil, false) + i.w.WriteHeader(utils.MethodNotAllowedForServiceCodeStr, nil, false, i.req) err := errors.New("method is not allowed") logging.Logger.Error(err.Error()) return xICAPMetadata, err @@ -86,11 +87,11 @@ func (i *ICAPRequest) RequestInitialization() (string, error) { //adding important headers to options ICAP response requiredService := service.GetService(i.vendor, i.serviceName, i.methodName, - &http_message.HttpMsg{Request: i.req.Request, Response: i.req.Response}, xICAPMetadata) + &http_message.HttpMsg{Request: i.req.Request, Response: i.req.Response, StorageClient: i.req.StorageClient, StorageKey: i.req.StorageKey}, xICAPMetadata) logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "adding ISTAG Service Headers")) i.addingISTAGServiceHeaders(requiredService.ISTagValue()) - logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "checking if returning 24 to ICAP client is allowed or not")) + logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "checking if returning 204 to ICAP client is allowed or not")) i.Is204Allowed = i.is204Allowed(xICAPMetadata) i.isShadowServiceEnabled = config.AppCfg.ServicesInstances[i.serviceName].ShadowService @@ -120,15 +121,18 @@ func (i *ICAPRequest) RequestProcessing(xICAPMetadata string) { "processing ICAP request upon the service and method required")) partial := false if i.methodName != utils.ICAPModeOptions { - file := &bytes.Buffer{} - fileLen := 0 + var fileLen int64 = 0 if i.methodName == utils.ICAPModeResp { - io.Copy(file, i.req.Response.Body) - fileLen = file.Len() - i.req.Response.Header.Set(utils.ContentLength, strconv.Itoa(len(file.Bytes()))) - i.req.Response.Body = io.NopCloser(bytes.NewBuffer(file.Bytes())) + fileStorLen, err := i.req.StorageClient.Size(i.req.StorageKey) + if err != nil { + log.Println(err) + return + } + fileLen = fileStorLen + i.req.Response.Header.Set(utils.ContentLength, strconv.FormatInt(fileLen, 10)) + //i.req.Response.Body = io.NopCloser(bytes.NewBuffer(file.Bytes())) } else { if i.req.Method == utils.ICAPModeReq { @@ -200,13 +204,6 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { defer i.req.Request.Body.Close() defer i.req.OrgRequest.Body.Close() - } else { - defer i.req.Response.Body.Close() - //someString := "hello world nand hello go and more" - //r := strings.NewReader(someString) - - //defer Original_rsp.Body.Close() - } if i.req.Request == nil { i.req.Request = &http.Request{} @@ -215,7 +212,7 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "initialize the service by creating instance from the required service")) requiredService := service.GetService(i.vendor, i.serviceName, i.methodName, - &http_message.HttpMsg{Request: i.req.Request, Response: i.req.Response}, xICAPMetadata) + &http_message.HttpMsg{Request: i.req.Request, Response: i.req.Response, StorageClient: i.req.StorageClient, StorageKey: i.req.StorageKey}, xICAPMetadata) logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "calling Processing func to process the http message which encapsulated inside the ICAP request")) @@ -226,6 +223,7 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { //icap.Request.Response IcapStatusCode, httpMsg, serviceHeaders, httpMshHeadersBeforeProcessing, httpMshHeadersAfterProcessing, vendorMsgs := requiredService.Processing(partial, i.req.Header) + defer i.req.StorageClient.Delete(i.req.StorageKey) // Delete the file in the tmp directory after processing(useful when error happens) // adding the headers which the service wants to add them in the ICAP response logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, @@ -251,7 +249,7 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { case utils.InternalServerErrStatusCodeStr: logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.InternalServerErrStatusCodeStr))) - i.w.WriteHeader(IcapStatusCode, nil, false) + i.w.WriteHeader(IcapStatusCode, nil, false, i.req) case utils.Continue: logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.Continue))) @@ -263,7 +261,8 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { i.req.Request.Body = io.NopCloser(bytes.NewBuffer(httpMsgBody.Bytes())) i.req.OrgRequest.Body = io.NopCloser(bytes.NewBuffer(httpMsgBody.Bytes())) } else { - i.req.Response.Body = io.NopCloser(bytes.NewBuffer(httpMsgBody.Bytes())) + // i.req.Response.Body = io.NopCloser(bytes.NewBuffer(httpMsgBody.Bytes())) + // i.req.StorageClient.Save(i.req.StorageKey, httpMsgBody.Bytes()) } i.allHeaders(IcapStatusCode, httpMshHeadersBeforeProcessing, httpMshHeadersAfterProcessing, vendorMsgs, xICAPMetadata) @@ -271,12 +270,12 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { case utils.RequestTimeOutStatusCodeStr: logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.RequestTimeOutStatusCodeStr))) - i.w.WriteHeader(IcapStatusCode, nil, false) + i.w.WriteHeader(IcapStatusCode, nil, false, i.req) case utils.NoModificationStatusCodeStr: logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.NoModificationStatusCodeStr))) if i.Is204Allowed { - i.w.WriteHeader(utils.NoModificationStatusCodeStr, nil, false) + i.w.WriteHeader(utils.NoModificationStatusCodeStr, nil, false, i.req) } else { IcapStatusCode = utils.OkStatusCodeStr @@ -286,10 +285,10 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { i.req.Request.Body = io.NopCloser(bytes.NewBuffer(body)) i.req.Request.Header.Set(utils.ContentLength, strconv.Itoa(len(body))) defer i.req.Request.Body.Close() - i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Request, true) + i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Request, true, i.req) } else { IcapStatusCode = utils.OkStatusCodeStr - i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true) + i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true, i.req) } //i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true) @@ -297,11 +296,11 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { case utils.OkStatusCodeStr: logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.OkStatusCodeStr))) - i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true) + i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true, i.req) case utils.BadRequestStatusCodeStr: logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.BadRequestStatusCodeStr))) - i.w.WriteHeader(IcapStatusCode, httpMsg, true) + i.w.WriteHeader(IcapStatusCode, httpMsg, true, i.req) } i.allHeaders(IcapStatusCode, httpMshHeadersBeforeProcessing, httpMshHeadersAfterProcessing, vendorMsgs, xICAPMetadata) } @@ -433,18 +432,18 @@ func (i *ICAPRequest) shadowService(xICAPMetadata string) { i.h["X-ICAPeg-Shadow-Service"] = []string{"true"} } if i.Is204Allowed { // following RFC3507, if the request has Allow: 204 header, it is to be checked and if it doesn't exists, return the request as it is to the ICAP client, https://tools.ietf.org/html/rfc3507#section-4.6 - i.w.WriteHeader(utils.NoModificationStatusCodeStr, nil, false) + i.w.WriteHeader(utils.NoModificationStatusCodeStr, nil, false, i.req) } else { if i.req.Method == "REQMOD" { - i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Request, true) + i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Request, true, i.req) tempBody, _ := io.ReadAll(i.req.Request.Body) i.w.Write(tempBody) i.req.Request.Body = io.NopCloser(bytes.NewBuffer(tempBody)) } else if i.req.Method == "RESPMOD" { - i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Response, true) - tempBody, _ := io.ReadAll(i.req.Response.Body) + i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Response, true, i.req) + tempBody, _ := i.req.StorageClient.Load(i.req.StorageKey) i.w.Write(tempBody) - i.req.Response.Body = io.NopCloser(bytes.NewBuffer(tempBody)) + // i.req.Response.Body = io.NopCloser(bytes.NewBuffer(tempBody)) } } } @@ -485,7 +484,7 @@ func (i *ICAPRequest) optionsMode(serviceName, xICAPMetadata string) { } } i.h.Set("Transfer-Preview", utils.Any) - i.w.WriteHeader(http.StatusOK, nil, false) + i.w.WriteHeader(http.StatusOK, nil, false, i.req) i.optionsRespHeaders = i.LogICAPResHeaders(http.StatusOK) } @@ -495,10 +494,19 @@ func (i *ICAPRequest) preview(xICAPMetadata string) *bytes.Buffer { logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "getting the rest of the body from client after the service returned ICAP "+ "response with status code"+strconv.Itoa(utils.Continue))) - r := icap.GetTheRest() - c := io.NopCloser(r) + reader := i.req.GetTheRest() + readCloser := io.NopCloser(reader) + defer readCloser.Close() + if i.req.Method == utils.ICAPModeResp { + err := i.req.StorageClient.AppendFromReader(i.req.StorageKey, readCloser) + if err != nil { + log.Println(err) + } + return nil + } + buf := new(bytes.Buffer) - buf.ReadFrom(c) + buf.ReadFrom(readCloser) return buf } diff --git a/block-page.html b/block-page.html index e852200a..36d6c55f 100644 --- a/block-page.html +++ b/block-page.html @@ -127,6 +127,7 @@

{{.Reason}}

diff --git a/config.toml b/config.toml index a7ace981..68d035aa 100644 --- a/config.toml +++ b/config.toml @@ -25,6 +25,8 @@ services= ["echo", "clhashlookup", "clamav","hashlocal"] debugging_headers=true web_server_host = "$_WEB_SERVER_HOST" #Example: "localhost:8081" , replace localhost with the ICAP server IP address. web_server_endpoint = "/service/message" +max_file_size_on_memory = 1 # megabytes +write_path_on_disk="./tmp" [echo] vendor = "echo" diff --git a/config/config.go b/config/config.go index b21518ff..e40f6b57 100644 --- a/config/config.go +++ b/config/config.go @@ -22,16 +22,18 @@ type serviceIcapInfo struct { // AppConfig represents the app configuration type AppConfig struct { - Port int - LogLevel string - WriteLogsToConsole bool - BypassExtensions []string - ProcessExtensions []string - PreviewBytes string - PreviewEnabled bool - DebuggingHeaders bool - Services []string - ServicesInstances map[string]*serviceIcapInfo + Port int + LogLevel string + WriteLogsToConsole bool + BypassExtensions []string + ProcessExtensions []string + PreviewBytes string + PreviewEnabled bool + DebuggingHeaders bool + Services []string + ServicesInstances map[string]*serviceIcapInfo + MaxFileSizeOnMemory int + WritePathOnDisk string } var AppCfg AppConfig @@ -48,11 +50,13 @@ func Init() { fmt.Println("app section doesn't exist in config file") } AppCfg = AppConfig{ - Port: readValues.ReadValuesInt("app.port"), - LogLevel: readValues.ReadValuesString("app.log_level"), - WriteLogsToConsole: readValues.ReadValuesBool("app.write_logs_to_console"), - DebuggingHeaders: readValues.ReadValuesBool("app.debugging_headers"), - Services: readValues.ReadValuesSlice("app.services"), + Port: readValues.ReadValuesInt("app.port"), + LogLevel: readValues.ReadValuesString("app.log_level"), + WriteLogsToConsole: readValues.ReadValuesBool("app.write_logs_to_console"), + DebuggingHeaders: readValues.ReadValuesBool("app.debugging_headers"), + Services: readValues.ReadValuesSlice("app.services"), + MaxFileSizeOnMemory: readValues.ReadValuesInt("app.max_file_size_on_memory"), + WritePathOnDisk: readValues.ReadValuesString("app.write_path_on_disk"), } logging.InitializeLogger(AppCfg.LogLevel, AppCfg.WriteLogsToConsole) logging.Logger.Info("Reading config.toml file") diff --git a/consts/consts.go b/consts/consts.go index 9c51f47e..f0e00e11 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -34,7 +34,7 @@ const ( ProcessExts = "process" RejectExts = "reject" BypassExts = "bypass" - BlockPagePath = "block-page.html" + BlockPagePath = "temp/exception-page.html" ErrPageReasonFileRejected = "fileRejected" ErrPageReasonMaxFileExceeded = "maxFileSizeExceeded" ErrPageReasonFileIsNotSafe = "fileIsNotSafe" diff --git a/go.mod b/go.mod index 711a69f4..8a76cb81 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/davecgh/go-spew v1.1.1 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e + github.com/google/uuid v1.1.2 github.com/h2non/filetype v1.0.12 github.com/spf13/viper v1.9.0 github.com/xhit/go-str2duration/v2 v2.0.0 diff --git a/go.sum b/go.sum index 3948ce7c..de522144 100644 --- a/go.sum +++ b/go.sum @@ -152,6 +152,7 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= diff --git a/http-message/httpMessage.go b/http-message/httpMessage.go index 089b17b3..d5ea2416 100644 --- a/http-message/httpMessage.go +++ b/http-message/httpMessage.go @@ -2,21 +2,26 @@ package http_message import ( "icapeg/logging" + "icapeg/storage" "net/http" ) // HttpMsg is a struct used for encapsulating http message (http request, http response) // to facilitate passing them together throw functions type HttpMsg struct { - Request *http.Request - Response *http.Response + Request *http.Request + Response *http.Response + StorageClient storage.StorageClient + StorageKey string } // NewHttpMsg is a func used for creating an instance from HttpMsg struct -func (h *HttpMsg) NewHttpMsg(Request *http.Request, Response *http.Response) *HttpMsg { +func (h *HttpMsg) NewHttpMsg(Request *http.Request, Response *http.Response, StorageClient storage.StorageClient, StorageKey string) *HttpMsg { logging.Logger.Debug("creating instance from HttpMsg struct") return &HttpMsg{ - Request: Request, - Response: Response, + Request: Request, + Response: Response, + StorageClient: StorageClient, + StorageKey: StorageKey, } } diff --git a/icap-client/client_test.go b/icap-client/client_test.go index e5e73644..0279c481 100644 --- a/icap-client/client_test.go +++ b/icap-client/client_test.go @@ -37,7 +37,7 @@ package icapclient // "Content-Length": []string{"19"}, // }, // ContentLength: 19, -// Body: ioutil.NopCloser(strings.NewReader("This is a GOOD FILE")), +// Body: io.NopCloser(strings.NewReader("This is a GOOD FILE")), // }, // wantedStatusCode: http.StatusNoContent, // wantedStatus: "No Modifications", @@ -58,7 +58,7 @@ package icapclient // "Content-Length": []string{"18"}, // }, // ContentLength: 18, -// Body: ioutil.NopCloser(strings.NewReader("This is a BAD FILE")), +// Body: io.NopCloser(strings.NewReader("This is a BAD FILE")), // }, // wantedStatusCode: http.StatusOK, // wantedStatus: "OK", @@ -233,7 +233,7 @@ package icapclient // "Content-Length": []string{"41"}, // }, // ContentLength: 41, -// Body: ioutil.NopCloser(strings.NewReader("Hello World!This is a GOOD FILE! bye bye!")), +// Body: io.NopCloser(strings.NewReader("Hello World!This is a GOOD FILE! bye bye!")), // }, // wantedStatusCode: http.StatusNoContent, // wantedStatus: "No Modifications", @@ -259,7 +259,7 @@ package icapclient // "Content-Length": []string{"18"}, // }, // ContentLength: 18, -// Body: ioutil.NopCloser(strings.NewReader("This is a BAD FILE")), +// Body: io.NopCloser(strings.NewReader("This is a BAD FILE")), // }, // wantedStatusCode: http.StatusOK, // wantedStatus: "OK", diff --git a/icap-client/test_server.go b/icap-client/test_server.go index 9f451165..7f49eb16 100644 --- a/icap-client/test_server.go +++ b/icap-client/test_server.go @@ -70,12 +70,12 @@ func respmodHandler(w icap.ResponseWriter, req *icap.Request) { h.Set("Preview", strconv.Itoa(previewBytes)) } h.Set("Transfer-Preview", "*") - w.WriteHeader(http.StatusOK, nil, false) + w.WriteHeader(http.StatusOK, nil, false, nil) case "RESPMOD": defer req.Response.Body.Close() if val, exist := req.Header["Allow"]; !exist || (len(val) > 0 && val[0] != "204") { - w.WriteHeader(http.StatusNoContent, nil, false) + w.WriteHeader(http.StatusNoContent, nil, false, nil) return } @@ -85,7 +85,7 @@ func respmodHandler(w icap.ResponseWriter, req *icap.Request) { if _, err := io.Copy(buf, req.Response.Body); err != nil { log.Println("Failed to copy the response body to buffer: ", err.Error()) - w.WriteHeader(http.StatusNoContent, nil, false) + w.WriteHeader(http.StatusNoContent, nil, false, nil) return } @@ -98,7 +98,7 @@ func respmodHandler(w icap.ResponseWriter, req *icap.Request) { status = http.StatusOK } - w.WriteHeader(status, nil, false) + w.WriteHeader(status, nil, false, nil) } } @@ -116,11 +116,11 @@ func reqmodHandler(w icap.ResponseWriter, req *icap.Request) { h.Set("Preview", strconv.Itoa(previewBytes)) } h.Set("Transfer-Preview", "*") - w.WriteHeader(http.StatusOK, nil, false) + w.WriteHeader(http.StatusOK, nil, false, nil) case "REQMOD": if val, exist := req.Header["Allow"]; !exist || (len(val) > 0 && val[0] != "204") { - w.WriteHeader(http.StatusNoContent, nil, false) + w.WriteHeader(http.StatusNoContent, nil, false, nil) return } @@ -137,7 +137,7 @@ func reqmodHandler(w icap.ResponseWriter, req *icap.Request) { status = http.StatusOK } - w.WriteHeader(status, nil, false) + w.WriteHeader(status, nil, false, nil) } } diff --git a/icap/bridge.go b/icap/bridge.go index f26aa8c0..a28dec89 100644 --- a/icap/bridge.go +++ b/icap/bridge.go @@ -52,7 +52,7 @@ func (w *bridgedRespWriter) WriteHeader(code int) { resp.StatusCode = code resp.Header = w.header - w.irw.WriteHeader(200, resp, true) + w.irw.WriteHeader(200, resp, true, nil) } // NewBridgedResponseWriter Create an http.ResponseWriter that encapsulates its response in an ICAP response. diff --git a/icap/mux.go b/icap/mux.go index 94561009..53f68300 100644 --- a/icap/mux.go +++ b/icap/mux.go @@ -80,7 +80,7 @@ func (mux *ServeMux) ServeICAP(w ResponseWriter, r *Request) { // Clean path to canonical form and redirect. if p := cleanPath(r.URL.Path); p != r.URL.Path { w.Header().Set("Location", p) - w.WriteHeader(http.StatusMovedPermanently, nil, false) + w.WriteHeader(http.StatusMovedPermanently, nil, false, r) return } // Host-specific pattern takes precedence over generic ones @@ -129,11 +129,11 @@ func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { // NotFound replies to the request with an HTTP 404 not found error. func NotFound(w ResponseWriter, r *Request) { - w.WriteHeader(http.StatusNotFound, nil, false) + w.WriteHeader(http.StatusNotFound, nil, false, nil) } // NotFoundHandler returns a simple request handler -// that replies to each request with a ``404 page not found'' reply. +// that replies to each request with a “404 page not found” reply. func NotFoundHandler() Handler { return HandlerFunc(NotFound) } // Redirect to a fixed URL @@ -189,5 +189,5 @@ func Redirect(w ResponseWriter, r *Request, redirectURL string, code int) { } w.Header().Set("Location", redirectURL) - w.WriteHeader(code, nil, false) + w.WriteHeader(code, nil, false, r) } diff --git a/icap/request.go b/icap/request.go index 865103a1..fec152b1 100644 --- a/icap/request.go +++ b/icap/request.go @@ -11,12 +11,16 @@ import ( "bufio" "bytes" "fmt" + "icapeg/config" + "icapeg/storage" "io" "net/http" "net/textproto" "net/url" "strconv" "strings" + + "github.com/google/uuid" ) type badStringError struct { @@ -39,13 +43,14 @@ type Request struct { EndIndicator string OrgRequest *http.Request // The HTTP messages. - Request *http.Request - Response *http.Response + Request *http.Request + Response *http.Response + OrigBuf *bufio.ReadWriter + OrigReader io.Reader + StorageClient storage.StorageClient + StorageKey string } -var origBuf *bufio.ReadWriter -var origReader io.Reader - // ReadRequest reads and parses a request from b. func ReadRequest(b *bufio.ReadWriter) (req *Request, err error) { tp := textproto.NewReader(b.Reader) @@ -146,12 +151,13 @@ func ReadRequest(b *bufio.ReadWriter) (req *Request, err error) { } var bodyReader io.ReadCloser = emptyReader(0) + defer bodyReader.Close() if hasBody { if p := req.Header.Get("Preview"); p != "" { req.Preview, err = io.ReadAll(newChunkedReader(b)) - origBuf = b - origReader = bytes.NewBuffer(req.Preview) + req.OrigBuf = b + req.OrigReader = bytes.NewBuffer(req.Preview) req.EndIndicator = "0" if err != nil { if strings.Contains(err.Error(), "ieof") { @@ -213,9 +219,15 @@ func ReadRequest(b *bufio.ReadWriter) (req *Request, err error) { } if req.Method == "RESPMOD" { - req.Response.Body = bodyReader + // req.Response.Body = bodyReader + if req.StorageClient == nil { + req.StorageClient = storage.NewAutoStorage(int64(config.App().MaxFileSizeOnMemory), config.App().WritePathOnDisk) // 1<<20 1 MB max memory size, disk storage path "/tmp" + } + req.SaveResponseBody(bodyReader) } else { - req.Response.Body = emptyReader(0) + // req.Response.Body = emptyReader(0) + req.SaveResponseBody(emptyReader(0)) + } } @@ -256,6 +268,24 @@ func (c *continueReader) Read(p []byte) (n int, err error) { return c.cr.Read(p) } -func GetTheRest() io.Reader { - return io.MultiReader(origReader, &continueReader{buf: origBuf}) +func (req *Request) GetTheRest() io.Reader { + return io.MultiReader(req.OrigReader, &continueReader{buf: req.OrigBuf}) +} +func (r *Request) SaveResponseBody(reqBody io.ReadCloser) error { + body, err := io.ReadAll(reqBody) + if err != nil { + return err + } + defer reqBody.Close() + + r.StorageKey = generateStorageKey() + return r.StorageClient.Save(r.StorageKey, body) +} + +func (r *Request) LoadResponseBody() ([]byte, error) { + return r.StorageClient.Load(r.StorageKey) +} + +func generateStorageKey() string { + return uuid.New().String() } diff --git a/icap/response.go b/icap/response.go index 7eb431d9..3d46bf0b 100644 --- a/icap/response.go +++ b/icap/response.go @@ -38,7 +38,7 @@ type ResponseWriter interface { // Then it sends an HTTP header if httpMessage is not nil. // httpMessage may be an *http.Request or an *http.Response. // hasBody should be true if there will be calls to Write(), generating a message body. - WriteHeader(code int, httpMessage interface{}, hasBody bool) + WriteHeader(code int, httpMessage interface{}, hasBody bool, req *Request) } type respWriter struct { @@ -56,7 +56,7 @@ func (w *respWriter) Header() http.Header { func (w *respWriter) Write(p []byte) (n int, err error) { if !w.wroteHeader { - w.WriteHeader(http.StatusOK, nil, true) + w.WriteHeader(http.StatusOK, nil, true, w.req) } if w.cw == nil { @@ -71,7 +71,7 @@ func (w *respWriter) WriteRaw(p string) { w.wroteRaw = true } -func (w *respWriter) WriteHeader(code int, httpMessage interface{}, hasBody bool) { +func (w *respWriter) WriteHeader(code int, httpMessage interface{}, hasBody bool, req *Request) { if w.wroteHeader { log.Println("Called WriteHeader twice on the same connection") return @@ -145,8 +145,15 @@ func (w *respWriter) WriteHeader(code int, httpMessage interface{}, hasBody bool if hasBody { switch msg := httpMessage.(type) { case *http.Response: - requestBody, _ := io.ReadAll(msg.Body) - w.Write(requestBody) + if req != nil { + requestBody, _ := req.StorageClient.Load(req.StorageKey) + w.Write(requestBody) + req.StorageClient.Delete(req.StorageKey) + + } else { + requestBody, _ := io.ReadAll(msg.Body) + w.Write(requestBody) + } case *http.Request: requestBody, _ := io.ReadAll(msg.Body) w.Write(requestBody) @@ -158,7 +165,7 @@ func (w *respWriter) WriteHeader(code int, httpMessage interface{}, hasBody bool func (w *respWriter) finishRequest() { if !w.wroteHeader { - w.WriteHeader(http.StatusOK, nil, false) + w.WriteHeader(http.StatusOK, nil, false, w.req) } if w.cw != nil && !w.wroteRaw { @@ -168,6 +175,9 @@ func (w *respWriter) finishRequest() { } w.conn.buf.Flush() + if w.req != nil && w.req.StorageClient != nil { + w.req.StorageClient.Delete(w.req.StorageKey) + } } // httpRequestHeader returns the headers for an HTTP request diff --git a/icap/response_test.go b/icap/response_test.go index d7fdd3b3..f90dbedb 100644 --- a/icap/response_test.go +++ b/icap/response_test.go @@ -78,6 +78,6 @@ func HandleREQMOD2(w ResponseWriter, req *Request) { body, _ := io.ReadAll(req.Request.Body) newBody := string(body) + " ICAP powered!" - w.WriteHeader(200, req.Request, true) + w.WriteHeader(200, req.Request, true, nil) io.WriteString(w, newBody) } diff --git a/icap/server.go b/icap/server.go index f193f6ca..a83fedb9 100644 --- a/icap/server.go +++ b/icap/server.go @@ -11,6 +11,8 @@ import ( "bytes" "crypto/tls" "fmt" + "icapeg/config" + "icapeg/storage" "log" "net" "net/http" @@ -69,6 +71,9 @@ func (c *conn) readRequest() (w *respWriter, err error) { } if req == nil { req = new(Request) + if req.StorageClient == nil { + req.StorageClient = storage.NewAutoStorage(int64(config.App().MaxFileSizeOnMemory), config.App().WritePathOnDisk) // 1<<20 1 MB max memory size, disk storage path "/tmp" + } } else { req.RemoteAddr = c.remoteAddr } diff --git a/server/server.go b/server/server.go index 0ec13056..697ba08e 100644 --- a/server/server.go +++ b/server/server.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "os/signal" + "path/filepath" "strconv" "syscall" "time" @@ -14,6 +15,8 @@ import ( "icapeg/api" "icapeg/config" "icapeg/icap" + + "go.uber.org/zap" ) // https://github.com/k8-proxy/k8-rebuild-rest-api @@ -46,7 +49,31 @@ func StartServer() error { logging.Logger.Fatal(err.Error()) } }() - + // This block to remove the contents in tmp dir at the start of the project and if it doesn't exist create one + dirPath := config.App().WritePathOnDisk + + // Check if the directory exists, create if it doesn't + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + err := os.MkdirAll(dirPath, os.ModePerm) + if err != nil { + logging.Logger.Fatal("Failed to create directory", zap.Error(err)) + } + } + // Read the directory contents + entries, err := os.ReadDir(dirPath) + if err != nil { + logging.Logger.Fatal("Failed to read directory", zap.Error(err)) + } + + for _, entry := range entries { + if !entry.IsDir() { // Skip directories + err = os.Remove(filepath.Join(dirPath, entry.Name())) + if err != nil { + logging.Logger.Fatal(err.Error()) + } + } + } + // The block ends here ticker := time.NewTicker(10 * time.Second) go func() { for { diff --git a/service/services-utilities/general-functions/general-functions.go b/service/services-utilities/general-functions/general-functions.go index fa5f1881..a267bee6 100644 --- a/service/services-utilities/general-functions/general-functions.go +++ b/service/services-utilities/general-functions/general-functions.go @@ -52,9 +52,9 @@ func NewGeneralFunc(httpMsg *http_message.HttpMsg, xICAPMetadata string) *Genera } // CopyingFileToTheBuffer is a func which used for extracting a file from the body of the http message -func (f *GeneralFunc) CopyingFileToTheBuffer(methodName string) (*bytes.Buffer, ContentTypes.ContentType, error) { +func (f *GeneralFunc) CopyingFileToTheBuffer(methodName string) ([]byte, ContentTypes.ContentType, error) { logging.Logger.Info(utils.PrepareLogMsg(f.xICAPMetadata, "extracting the body of HTTP message")) - file := &bytes.Buffer{} + var file []byte var err error var reqContentType ContentTypes.ContentType reqContentType = nil @@ -63,7 +63,9 @@ func (f *GeneralFunc) CopyingFileToTheBuffer(methodName string) (*bytes.Buffer, file, reqContentType, err = f.copyingFileToTheBufferReq() break case utils.ICAPModeResp: - file, err = f.copyingFileToTheBufferResp() + // file, err = f.copyingFileToTheBufferResp() + // no need we will get file form storgeclient + break } if err != nil { @@ -92,7 +94,8 @@ func (f *GeneralFunc) CheckTheExtension(fileExtension string, extArrs []services if methodName == "RESPMOD" { errPage := f.GenHtmlPage(BlockPagePath, utils.ErrPageReasonFileRejected, serviceName, identifier, requestURI, fileSize, f.xICAPMetadata) f.httpMsg.Response = f.ErrPageResp(http.StatusForbidden, errPage.Len()) - f.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(errPage.Bytes())) + // f.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(errPage.Bytes())) + f.httpMsg.StorageClient.Save(f.httpMsg.StorageKey, errPage.Bytes()) return false, utils.OkStatusCodeStr, f.httpMsg.Response } else { htmlPage, req, err := f.ReqModErrPage(utils.ErrPageReasonFileRejected, serviceName, "-", fileSize) @@ -111,22 +114,23 @@ func (f *GeneralFunc) CheckTheExtension(fileExtension string, extArrs []services } else if extArrs[i].Name == utils.BypassExts { if f.ifFileExtIsX(fileExtension, bypassExts) { logging.Logger.Debug(utils.PrepareLogMsg(f.xICAPMetadata, "extension is bypass")) - fileAfterPrep, httpMsg := f.IfICAPStatusIs204(methodName, utils.NoModificationStatusCodeStr, - file, isGzip, reqContentType, f.httpMsg) - if fileAfterPrep == nil && httpMsg == nil { - return false, utils.InternalServerErrStatusCodeStr, nil - } - - //returning the http message and the ICAP status code - switch msg := httpMsg.(type) { - case *http.Request: - msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) - return false, utils.NoModificationStatusCodeStr, msg - case *http.Response: - msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) - return false, utils.NoModificationStatusCodeStr, msg - } return false, utils.NoModificationStatusCodeStr, nil + // fileAfterPrep, httpMsg := f.IfICAPStatusIs204(methodName, utils.NoModificationStatusCodeStr, + // file, isGzip, reqContentType, f.httpMsg) + // if fileAfterPrep == nil && httpMsg == nil { + // return false, utils.InternalServerErrStatusCodeStr, nil + // } + + // //returning the http message and the ICAP status code + // switch msg := httpMsg.(type) { + // case *http.Request: + // msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) + // return false, utils.NoModificationStatusCodeStr, msg + // case *http.Response: + // msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) + // return false, utils.NoModificationStatusCodeStr, msg + // } + // return false, utils.NoModificationStatusCodeStr, nil } } } @@ -135,19 +139,22 @@ func (f *GeneralFunc) CheckTheExtension(fileExtension string, extArrs []services // copyingFileToTheBufferResp is a utility function for CopyingFileToTheBuffer func // it's used for extracting a file from the body of the http response -func (f *GeneralFunc) copyingFileToTheBufferResp() (*bytes.Buffer, error) { - file := &bytes.Buffer{} - _, err := io.Copy(file, f.httpMsg.Response.Body) - return file, err +func (f *GeneralFunc) copyingFileToTheBufferResp() ([]byte, error) { + // _, err := io.Copy(file, f.httpMsg.Response.Body) + file, err := f.httpMsg.StorageClient.Load(f.httpMsg.StorageKey) + if err != nil { + return file, err + } + return file, nil } // copyingFileToTheBufferReq is a utility function for CopyingFileToTheBuffer func // it's used for extracting a file from the body of the http request -func (f *GeneralFunc) copyingFileToTheBufferReq() (*bytes.Buffer, ContentTypes.ContentType, error) { +func (f *GeneralFunc) copyingFileToTheBufferReq() ([]byte, ContentTypes.ContentType, error) { reqContentType := ContentTypes.GetContentType(f.httpMsg.Request) // getting the file from request and store it in buf as a type of bytes.Buffer file := reqContentType.GetFileFromRequest() - return file, reqContentType, nil + return file.Bytes(), reqContentType, nil } @@ -419,8 +426,10 @@ func (f *GeneralFunc) ReturningHttpMessageWithFile(methodName string, file []byt } return f.httpMsg.Request case utils.ICAPModeResp: - f.httpMsg.Response.Header.Set(utils.ContentLength, strconv.Itoa(len(string(file)))) - f.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(file)) + size, _ := f.httpMsg.StorageClient.Size(f.httpMsg.StorageKey) + f.httpMsg.Response.Header.Set(utils.ContentLength, strconv.Itoa(int(size))) + //f.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(file)) + //f.httpMsg.StorageClient.Save(f.httpMsg.StorageKey, file) return f.httpMsg.Response } return nil @@ -459,7 +468,8 @@ func (f *GeneralFunc) returningHttpMessage(methodName string, file []byte) inter return f.httpMsg.Request case utils.ICAPModeResp: f.httpMsg.Response.Header.Set(utils.ContentLength, strconv.Itoa(len(string(file)))) - f.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(file)) + // f.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(file)) + f.httpMsg.StorageClient.Save(f.httpMsg.StorageKey, file) return f.httpMsg.Response } return nil diff --git a/service/services/clamav/clamav.go b/service/services/clamav/clamav.go index 873a329c..8ae65023 100644 --- a/service/services/clamav/clamav.go +++ b/service/services/clamav/clamav.go @@ -62,7 +62,7 @@ func (c *Clamav) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, //if the http method is Connect, return the request as it is because it has no body if c.methodName == utils.ICAPModeReq { if c.httpMsg.Request.Method == http.MethodConnect { - return utils.OkStatusCodeStr, c.generalFunc.ReturningHttpMessageWithFile(c.methodName, file.Bytes()), + return utils.OkStatusCodeStr, c.generalFunc.ReturningHttpMessageWithFile(c.methodName, file), serviceHeaders, msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs } } @@ -84,22 +84,22 @@ func (c *Clamav) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, logging.Logger.Info(utils.PrepareLogMsg(c.xICAPMetadata, c.serviceName+" file name : "+fileName)) - fileExtension := c.generalFunc.GetMimeExtension(file.Bytes(), contentType[0], fileName) + fileExtension := c.generalFunc.GetMimeExtension(file, contentType[0], fileName) //check if the file extension is a bypass extension //if yes we will not modify the file, and we will return 204 No modifications hash := sha256.New() f := file - _, err = hash.Write(f.Bytes()) + _, err = hash.Write(f) if err != nil { fmt.Println(err.Error()) } - fileSize := fmt.Sprintf("%v", file.Len()) + fileSize := fmt.Sprintf("%v", len(file)) fileHash := hex.EncodeToString(hash.Sum([]byte(nil))) logging.Logger.Info(utils.PrepareLogMsg(c.xICAPMetadata, c.serviceName+" file hash : "+fileHash)) isProcess, icapStatus, httpMsg := c.generalFunc.CheckTheExtension(fileExtension, c.extArrs, c.processExts, c.rejectExts, c.bypassExts, c.return400IfFileExtRejected, isGzip, - c.serviceName, c.methodName, fileHash, c.httpMsg.Request.RequestURI, reqContentType, file, ExceptionPagePath, fileSize) + c.serviceName, c.methodName, fileHash, c.httpMsg.Request.RequestURI, reqContentType, bytes.NewBuffer(file), ExceptionPagePath, fileSize) if !isProcess { logging.Logger.Info(utils.PrepareLogMsg(c.xICAPMetadata, c.serviceName+" service has stopped processing")) msgHeadersAfterProcessing = c.generalFunc.LogHTTPMsgHeaders(c.methodName) @@ -109,8 +109,8 @@ func (c *Clamav) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, //check if the file size is greater than max file size of the service //if yes we will return 200 ok or 204 no modification, it depends on the configuration of the service - if c.maxFileSize != 0 && c.maxFileSize < file.Len() { - status, file, httpMsg := c.generalFunc.IfMaxFileSizeExc(c.returnOrigIfMaxSizeExc, c.serviceName, c.methodName, file, c.maxFileSize, ExceptionPagePath, fileSize) + if c.maxFileSize != 0 && c.maxFileSize < len(file) { + status, file, httpMsg := c.generalFunc.IfMaxFileSizeExc(c.returnOrigIfMaxSizeExc, c.serviceName, c.methodName, bytes.NewBuffer(file), c.maxFileSize, ExceptionPagePath, fileSize) fileAfterPrep, httpMsg := c.generalFunc.IfStatusIs204WithFile(c.methodName, status, file, isGzip, reqContentType, httpMsg, true) if fileAfterPrep == nil && httpMsg == nil { logging.Logger.Info(utils.PrepareLogMsg(c.xICAPMetadata, c.serviceName+" service has stopped processing")) @@ -139,7 +139,7 @@ func (c *Clamav) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, clmd := clamd.NewClamd(c.SocketPath) logging.Logger.Debug(utils.PrepareLogMsg(c.xICAPMetadata, "sending the HTTP msg body to the ClamAV through antivirus socket")) - response, err := clmd.ScanStream(bytes.NewReader(file.Bytes()), make(chan bool)) + response, err := clmd.ScanStream(bytes.NewReader(file), make(chan bool)) if err != nil { logging.Logger.Error(utils.PrepareLogMsg(c.xICAPMetadata, c.serviceName+" error: "+err.Error())) logging.Logger.Info(utils.PrepareLogMsg(c.xICAPMetadata, c.serviceName+" service has stopped processing")) @@ -177,10 +177,13 @@ func (c *Clamav) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, c.httpMsg.Response = c.generalFunc.ErrPageResp(c.CaseBlockHttpResponseCode, errPage.Len()) if c.CaseBlockHttpBody { - c.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(errPage.Bytes())) + // c.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(errPage.Bytes())) + c.httpMsg.StorageClient.Save(c.httpMsg.StorageKey, errPage.Bytes()) + } else { var r []byte - c.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(r)) + // c.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(r)) + c.httpMsg.StorageClient.Save(c.httpMsg.StorageKey, r) delete(c.httpMsg.Response.Header, "Content-Type") delete(c.httpMsg.Response.Header, "Content-Length") } @@ -205,7 +208,7 @@ func (c *Clamav) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, } //returning the scanned file if everything is ok fileAfterPrep, httpMsg := c.generalFunc.IfICAPStatusIs204(c.methodName, utils.NoModificationStatusCodeStr, - file, false, reqContentType, c.httpMsg) + bytes.NewBuffer(file), false, reqContentType, c.httpMsg) if fileAfterPrep == nil && httpMsg == nil { logging.Logger.Info(utils.PrepareLogMsg(c.xICAPMetadata, c.serviceName+" service has stopped processing")) return utils.InternalServerErrStatusCodeStr, nil, nil, msgHeadersBeforeProcessing, diff --git a/service/services/clhashlookup/clhashlookup.go b/service/services/clhashlookup/clhashlookup.go index ea04eb83..b9269b27 100644 --- a/service/services/clhashlookup/clhashlookup.go +++ b/service/services/clhashlookup/clhashlookup.go @@ -9,216 +9,279 @@ import ( "fmt" utils "icapeg/consts" "icapeg/logging" + "icapeg/service/services-utilities/ContentTypes" "io" "net/http" "net/textproto" + "path/filepath" "strconv" "strings" "time" ) -// Processing is a func used for to processing the http message -func (h *Hashlookup) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, interface{}, map[string]string, map[string]interface{}, - map[string]interface{}, map[string]interface{}) { - serviceHeaders := make(map[string]string) - serviceHeaders["X-ICAP-Metadata"] = h.xICAPMetadata - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has started processing")) - msgHeadersBeforeProcessing := h.generalFunc.LogHTTPMsgHeaders(h.methodName) - msgHeadersAfterProcessing := make(map[string]interface{}) - vendorMsgs := make(map[string]interface{}) - h.IcapHeaders = IcapHeader - h.IcapHeaders.Add("X-ICAP-Metadata", h.xICAPMetadata) - // no need to scan part of the file, this service needs all the file at ine time +func (d *Hashlookup) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + d.initProcessing(IcapHeader) + if partial { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, - h.serviceName+" service has stopped processing partially")) - return utils.Continue, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } - if h.methodName == utils.ICAPModeResp { - if h.httpMsg.Response != nil { - if h.httpMsg.Response.StatusCode == 206 { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing byte range received")) - return utils.NoModificationStatusCodeStr, h.httpMsg, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } - } + return d.handlePartialProcessing() + } + + file, reqContentType, err := d.extractFile() + if err != nil { + return d.handleError(utils.InternalServerErrStatusCodeStr, err) + } + + if d.methodName == utils.ICAPModeReq && d.httpMsg.Request.Method == http.MethodConnect { + return d.handleConnectMethod(file) } - isGzip := false + + fileName, contentType := d.getFileNameAndContentType() + fileSize := d.getFileSize(file) + ///////////////////////// ExceptionPagePath := utils.BlockPagePath + // fileExtension := d.generalFunc.GetMimeExtension(file, contentType[0], fileName) + var fileExtension string + filehead, err := d.httpMsg.StorageClient.ReadFileHeader(d.httpMsg.StorageKey) + if err != nil { + // Handle the error by extracting the file extension from the filename + logging.Logger.Warn(utils.PrepareLogMsg(d.xICAPMetadata, + "failed to read file header, falling back to file extension from filename: "+err.Error())) + fileExtension = filepath.Ext(fileName)[1:] + } else { + // Determine the file extension using the header data + fileExtension = d.generalFunc.GetMimeExtension(filehead, contentType[0], fileName) + } + d.FileHash, err = d.calculateFileHash(file) + if err != nil { + logging.Logger.Error(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" calculateFileHash error : "+err.Error())) + } + ///////////////////////////////// + isProcess, _, _ := d.generalFunc.CheckTheExtension(fileExtension, d.extArrs, + d.processExts, d.rejectExts, d.bypassExts, d.return400IfFileExtRejected, d.isGzip(), + d.serviceName, d.methodName, d.FileHash, d.httpMsg.Request.RequestURI, reqContentType, bytes.NewBuffer(file), ExceptionPagePath, fmt.Sprint(fileSize)) + + /////////////////////////////////// - if h.ExceptionPage != "" { - ExceptionPagePath = h.ExceptionPage + if d.isMaxFileSizeExceeded(fileSize) { + return d.handleMaxFileSizeExceeded(bytes.NewBuffer(file), fileSize, reqContentType) } - //extracting the file from http message - file, reqContentType, err := h.generalFunc.CopyingFileToTheBuffer(h.methodName) + if fileSize == 0 { + return d.handleEmptyFile(file) + } + isMal, err := d.sendFileToScan(d.FileHash, isProcess) if err != nil { - logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" error: "+err.Error())) - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return utils.InternalServerErrStatusCodeStr, nil, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs + return d.handleScanningError(err, file) } - //if the http method is Connect, return the request as it is because it has no body - if h.methodName == utils.ICAPModeReq { - if h.httpMsg.Request.Method == http.MethodConnect { - return utils.OkStatusCodeStr, h.generalFunc.ReturningHttpMessageWithFile(h.methodName, file.Bytes()), - serviceHeaders, msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } + if isMal { + return d.handleMaliciousFile(fileSize) } + return d.handleSuccessfulScan(file, fileSize, reqContentType) +} - //getting the extension of the file - var contentType []string +func (d *Hashlookup) initProcessing(IcapHeader textproto.MIMEHeader) { + d.serviceHeaders = map[string]string{"X-ICAP-Metadata": d.xICAPMetadata} + d.msgHeadersBeforeProcessing = d.generalFunc.LogHTTPMsgHeaders(d.methodName) + d.msgHeadersAfterProcessing = make(map[string]interface{}) + d.vendorMsgs = make(map[string]interface{}) + d.IcapHeaders = IcapHeader + d.IcapHeaders.Add("X-ICAP-Metadata", d.xICAPMetadata) + logging.Logger.Info(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" service has started processing")) +} + +func (d *Hashlookup) handlePartialProcessing() (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Info(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" service has stopped processing partially")) + return utils.Continue, nil, nil, d.msgHeadersBeforeProcessing, d.msgHeadersAfterProcessing, d.vendorMsgs +} + +func (d *Hashlookup) extractFile() ([]byte, ContentTypes.ContentType, error) { + file, reqContentType, err := d.generalFunc.CopyingFileToTheBuffer(d.methodName) + if err != nil { + logging.Logger.Error(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" error: "+err.Error())) + logging.Logger.Info(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" service has stopped processing")) + } + return file, reqContentType, err +} + +func (d *Hashlookup) handleError(status int, err error) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Error(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" error: "+err.Error())) + logging.Logger.Info(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" service has stopped processing")) + return status, nil, d.serviceHeaders, d.msgHeadersBeforeProcessing, d.msgHeadersAfterProcessing, d.vendorMsgs +} + +func (d *Hashlookup) handleConnectMethod(file []byte) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + return utils.OkStatusCodeStr, d.generalFunc.ReturningHttpMessageWithFile(d.methodName, file), d.serviceHeaders, d.msgHeadersBeforeProcessing, d.msgHeadersAfterProcessing, d.vendorMsgs +} +func (d *Hashlookup) getFileNameAndContentType() (string, []string) { + var contentType []string var fileName string - if h.methodName == utils.ICAPModeReq { - contentType = h.httpMsg.Request.Header["Content-Type"] - fileName = h.generalFunc.GetFileName(h.serviceName, h.xICAPMetadata) + if d.methodName == utils.ICAPModeReq { + contentType = d.httpMsg.Request.Header["Content-Type"] + fileName = d.generalFunc.GetFileName(d.serviceName, d.xICAPMetadata) } else { - contentType = h.httpMsg.Response.Header["Content-Type"] - fileName = h.generalFunc.GetFileName(h.serviceName, h.xICAPMetadata) + contentType = d.httpMsg.Response.Header["Content-Type"] + fileName = d.generalFunc.GetFileName(d.serviceName, d.xICAPMetadata) } if len(contentType) == 0 { contentType = append(contentType, "") } + if filepath.Ext(fileName) == "" { + fileName += "." + utils.Unknown + } + if filepath.Ext(fileName) == "." { + fileName += utils.Unknown + } + logging.Logger.Info(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" file name : "+fileName)) + return fileName, contentType +} - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" file name : "+fileName)) +func (d *Hashlookup) getFileSize(file []byte) int { + if d.methodName == utils.ICAPModeResp { + size, _ := d.httpMsg.StorageClient.Size(d.httpMsg.StorageKey) + return int(size) + } + return len(file) +} - fileExtension := h.generalFunc.GetMimeExtension(file.Bytes(), contentType[0], fileName) - //check if the file extension is a bypass extension - //if yes we will not modify the file, and we will return 204 No modifications +func (d *Hashlookup) isMaxFileSizeExceeded(fileSize int) bool { + return d.maxFileSize != 0 && d.maxFileSize < fileSize +} - hash := sha256.New() - f := file - _, err = hash.Write(f.Bytes()) - if err != nil { - fmt.Println(err.Error()) +func (d *Hashlookup) handleMaxFileSizeExceeded(file *bytes.Buffer, fileSize int, reqContentType ContentTypes.ContentType) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + status, file, httpMsg := d.generalFunc.IfMaxFileSizeExc(d.returnOrigIfMaxSizeExc, d.serviceName, d.methodName, file, d.maxFileSize, d.ExceptionPagePath(), fmt.Sprint(fileSize)) + fileAfterPrep, httpMsg := d.generalFunc.IfStatusIs204WithFile(d.methodName, status, file, d.isGzip(), reqContentType, httpMsg, true) + if fileAfterPrep == nil && httpMsg == nil { + return d.handleError(utils.InternalServerErrStatusCodeStr, fmt.Errorf("fileAfterPrep bytes is null")) } - fileSize := fmt.Sprintf("%v", file.Len()) - fileHash := hex.EncodeToString(hash.Sum([]byte(nil))) - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" file hash : "+fileHash)) + return d.prepareHttpResponse(status, httpMsg, fileAfterPrep) +} - //check if the file extension is a bypass extension - //if yes we will not modify the file, and we will return 204 No modifications - isProcess, icapStatus, httpMsg := h.generalFunc.CheckTheExtension(fileExtension, h.extArrs, - h.processExts, h.rejectExts, h.bypassExts, h.return400IfFileExtRejected, isGzip, - h.serviceName, h.methodName, fileHash, h.httpMsg.Request.RequestURI, reqContentType, file, ExceptionPagePath, fileSize) - if !isProcess { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return icapStatus, httpMsg, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } - //check if the file size is greater than max file size of the service - //if yes we will return 200 ok or 204 no modification, it depends on the configuration of the service - if h.maxFileSize != 0 && h.maxFileSize < file.Len() { - status, file, httpMsg := h.generalFunc.IfMaxFileSizeExc(h.returnOrigIfMaxSizeExc, h.serviceName, h.methodName, file, h.maxFileSize, ExceptionPagePath, fileSize) - fileAfterPrep, httpMsg := h.generalFunc.IfStatusIs204WithFile(h.methodName, status, file, isGzip, reqContentType, httpMsg, true) - if fileAfterPrep == nil && httpMsg == nil { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return utils.InternalServerErrStatusCodeStr, nil, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } - switch msg := httpMsg.(type) { - case *http.Request: - msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return status, msg, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - case *http.Response: - msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return status, msg, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs +func (d *Hashlookup) handleEmptyFile(file []byte) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Info(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" service has stopped processing zero file length")) + return utils.NoModificationStatusCodeStr, d.generalFunc.ReturningHttpMessageWithFile(d.methodName, file), d.serviceHeaders, d.msgHeadersBeforeProcessing, d.msgHeadersAfterProcessing, d.vendorMsgs +} + +func (d *Hashlookup) handleScanningError(err error, file []byte) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Error(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" error: "+err.Error())) + if !d.BypassOnApiError { + if strings.Contains(err.Error(), "context deadline exceeded") { + return d.handleError(utils.RequestTimeOutStatusCodeStr, err) } - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return status, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs + return d.handleError(utils.BadRequestStatusCodeStr, err) } + logging.Logger.Info(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" BypassOnApiError true")) + return utils.NoModificationStatusCodeStr, d.generalFunc.ReturningHttpMessageWithFile(d.methodName, file), d.serviceHeaders, d.msgHeadersBeforeProcessing, d.msgHeadersAfterProcessing, d.vendorMsgs +} - scannedFile := file.Bytes() - isMal, err := h.sendFileToScan(file) - if err != nil && !h.BypassOnApiError { - logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" error: "+err.Error())) - if strings.Contains(err.Error(), "context deadline exceeded") { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return utils.RequestTimeOutStatusCodeStr, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs +func (d *Hashlookup) handleMaliciousFile(fileSize int) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Debug(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+": file is not safe")) + if d.methodName == utils.ICAPModeResp { + serviceHeadersStr := mapToString(d.serviceHeaders) + errPage := d.generalFunc.GenHtmlPage(d.ExceptionPagePath(), utils.ErrPageReasonFileIsNotSafe, d.serviceName, d.FileHash, d.httpMsg.Request.RequestURI, fmt.Sprint(fileSize), serviceHeadersStr) + d.httpMsg.Response = d.generalFunc.ErrPageResp(d.CaseBlockHttpResponseCode, errPage.Len()) + d.saveErrorPageBody(errPage.Bytes()) + return d.prepareHttpResponse(utils.OkStatusCodeStr, d.httpMsg.Response, nil) + } else { + htmlPage, req, err := d.generalFunc.ReqModErrPage(utils.ErrPageReasonFileIsNotSafe, d.serviceName, d.FileHash, fmt.Sprint(fileSize)) + if err != nil { + return d.handleError(utils.InternalServerErrStatusCodeStr, err) } - // its suppose to be InternalServerErrStatusCodeStr but need to be handled - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return utils.BadRequestStatusCodeStr, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs + req.Body = io.NopCloser(htmlPage) + return d.prepareHttpResponse(utils.OkStatusCodeStr, req, nil) } +} - if isMal { - logging.Logger.Debug(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+": file is not safe")) - if h.methodName == utils.ICAPModeResp { - - errPage := h.generalFunc.GenHtmlPage(ExceptionPagePath, utils.ErrPageReasonFileIsNotSafe, h.serviceName, h.FileHash, h.httpMsg.Request.RequestURI, fileSize, h.xICAPMetadata) - - h.httpMsg.Response = h.generalFunc.ErrPageResp(h.CaseBlockHttpResponseCode, errPage.Len()) - if h.CaseBlockHttpBody { - h.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(errPage.Bytes())) - } else { - var r []byte - h.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(r)) - delete(h.httpMsg.Response.Header, "Content-Type") - delete(h.httpMsg.Response.Header, "Content-Length") +func (d *Hashlookup) handleSuccessfulScan(scannedFile []byte, fileSize int, reqContentType ContentTypes.ContentType) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + // Log headers and processing stop + d.generalFunc.LogHTTPMsgHeaders(d.methodName) + logging.Logger.Info(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" service has stopped processing")) + d.msgHeadersAfterProcessing = d.generalFunc.LogHTTPMsgHeaders(d.methodName) + + // Process the scanned file + scannedFile = d.generalFunc.PreparingFileAfterScanning(scannedFile, reqContentType, d.methodName) + + return utils.NoModificationStatusCodeStr, d.generalFunc.ReturningHttpMessageWithFile(d.methodName, scannedFile), + d.serviceHeaders, d.msgHeadersBeforeProcessing, d.msgHeadersAfterProcessing, d.vendorMsgs +} + +func (d *Hashlookup) saveErrorPageBody(body []byte) { + if d.CaseBlockHttpBody { + d.httpMsg.StorageClient.Save(d.httpMsg.StorageKey, body) + } else { + var r []byte + d.httpMsg.StorageClient.Save(d.httpMsg.StorageKey, r) + delete(d.httpMsg.Response.Header, "Content-Type") + delete(d.httpMsg.Response.Header, "Content-Length") + } +} +func (d *Hashlookup) prepareHttpResponse(status int, httpMsg interface{}, fileAfterPrep []byte) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + if httpMsg != nil { + switch msg := httpMsg.(type) { + case *http.Request: + if fileAfterPrep != nil { + msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) } - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return utils.OkStatusCodeStr, h.httpMsg.Response, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } else { - htmlPage, req, err := h.generalFunc.ReqModErrPage(utils.ErrPageReasonFileIsNotSafe, h.serviceName, h.FileHash, fileSize) - if err != nil { - logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" error: "+err.Error())) - - return utils.InternalServerErrStatusCodeStr, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs + case *http.Response: + if fileAfterPrep != nil { + msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) } - req.Body = io.NopCloser(htmlPage) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return utils.OkStatusCodeStr, req, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs } } + d.msgHeadersAfterProcessing = d.generalFunc.LogHTTPMsgHeaders(d.methodName) + logging.Logger.Info(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" service has stopped processing")) + return status, httpMsg, d.serviceHeaders, d.msgHeadersBeforeProcessing, d.msgHeadersAfterProcessing, d.vendorMsgs +} + +func (d *Hashlookup) ExceptionPagePath() string { + if d.ExceptionPage != "" { + return d.ExceptionPage + } + return utils.BlockPagePath +} - //returning the scanned file if everything is ok - //scannedFile = h.generalFunc.PreparingFileAfterScanning(scannedFile, reqContentType, h.methodName) - h.generalFunc.LogHTTPMsgHeaders(h.methodName) - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - // OkStatusCodeStr => NoModificationStatusCodeStr - /* - return utils.NoModificationStatusCodeStr, - h.generalFunc.ReturningHttpMessageWithFile(h.methodName, scannedFile), serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - /*return utils.NoModificationStatusCodeStr, nil, serviceHeaders, msgHeadersBeforeProcessing, - msgHeadersAfterProcessing, vendorMsgs */ - /*return utils.NoModificationStatusCodeStr, - h.ReturningHttpMessageWithFile(h.methodName, scannedFile, h.OriginalMsg), serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs*/ - scannedFile = h.generalFunc.PreparingFileAfterScanning(scannedFile, reqContentType, h.methodName) +func (d *Hashlookup) isGzip() bool { + return false +} - return utils.NoModificationStatusCodeStr, h.generalFunc.ReturningHttpMessageWithFile(h.methodName, scannedFile), - serviceHeaders, msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs +// New function to calculate file hash without consuming memory +func (d *Hashlookup) calculateFileHash(file []byte) (string, error) { + var fileHash string + if d.methodName == utils.ICAPModeReq || d.httpMsg.StorageClient == nil { + hash := sha256.New() + _, err := hash.Write(file) + if err != nil { + logging.Logger.Error(utils.PrepareLogMsg(d.xICAPMetadata, d.serviceName+" calculateFileHash error : "+err.Error())) + + } + fileHash = hex.EncodeToString(hash.Sum([]byte(nil))) + return fileHash, nil + } + + return d.httpMsg.StorageClient.ComputeHash(d.httpMsg.StorageKey) +} + +func mapToString(m map[string]string) string { + var sb strings.Builder + for key, value := range m { + sb.WriteString(fmt.Sprintf("\r\n")) + sb.WriteString(fmt.Sprintf("%s: %s", key, value)) + } + return sb.String() } // SendFileToScan is a function to send the file to API -func (h *Hashlookup) sendFileToScan(f *bytes.Buffer) (bool, error) { - hash := sha256.New() - _, _ = io.Copy(hash, f) - fileHash := hex.EncodeToString(hash.Sum([]byte(nil))) - h.FileHash = fileHash +func (h *Hashlookup) sendFileToScan(f string, isProcess bool) (bool, error) { + + if !isProcess { + return false, nil + } + h.FileHash = f //var jsonStr = []byte(`{"hash":"` + fileHash + `"}`) - req, err := http.NewRequest("GET", h.ScanUrl+fileHash, nil) + req, err := http.NewRequest("GET", h.ScanUrl+h.FileHash, nil) client := &http.Client{} ctx, cancel := context.WithTimeout(context.Background(), h.Timeout) defer cancel() diff --git a/service/services/clhashlookup/config.go b/service/services/clhashlookup/config.go index 1132a86f..520dba15 100644 --- a/service/services/clhashlookup/config.go +++ b/service/services/clhashlookup/config.go @@ -37,6 +37,10 @@ type Hashlookup struct { CaseBlockHttpBody bool ExceptionPage string IcapHeaders textproto.MIMEHeader + serviceHeaders map[string]string + msgHeadersBeforeProcessing map[string]interface{} + msgHeadersAfterProcessing map[string]interface{} + vendorMsgs map[string]interface{} } func InitHashlookupConfig(serviceName string) { diff --git a/service/services/echo/echo.go b/service/services/echo/echo.go index 5499a591..8b5de0de 100644 --- a/service/services/echo/echo.go +++ b/service/services/echo/echo.go @@ -41,7 +41,7 @@ func (e *Echo) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, i //if the http method is Connect, return the request as it is because it has no body if e.httpMsg.Request.Method == http.MethodConnect { - return utils.OkStatusCodeStr, e.generalFunc.ReturningHttpMessageWithFile(e.methodName, file.Bytes()), + return utils.OkStatusCodeStr, e.generalFunc.ReturningHttpMessageWithFile(e.methodName, file), serviceHeaders, msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs } @@ -61,14 +61,14 @@ func (e *Echo) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, i if len(contentType) == 0 { contentType = append(contentType, "") } - fileExtension := e.generalFunc.GetMimeExtension(file.Bytes(), contentType[0], fileName) - fileSize := fmt.Sprintf("%v kb", file.Len()/1000) + fileExtension := e.generalFunc.GetMimeExtension(file, contentType[0], fileName) + fileSize := fmt.Sprintf("%v kb", len(file)/1000) //check if the file extension is a bypass extension //if yes we will not modify the file, and we will return 204 No modifications isProcess, icapStatus, httpMsg := e.generalFunc.CheckTheExtension(fileExtension, e.extArrs, e.processExts, e.rejectExts, e.bypassExts, e.return400IfFileExtRejected, isGzip, - e.serviceName, e.methodName, EchoIdentifier, e.httpMsg.Request.RequestURI, reqContentType, file, utils.BlockPagePath, fileSize) + e.serviceName, e.methodName, EchoIdentifier, e.httpMsg.Request.RequestURI, reqContentType, bytes.NewBuffer(file), utils.BlockPagePath, fileSize) if !isProcess { logging.Logger.Info(utils.PrepareLogMsg(e.xICAPMetadata, e.serviceName+" service has stopped processing")) msgHeadersAfterProcessing = e.generalFunc.LogHTTPMsgHeaders(e.methodName) @@ -78,8 +78,8 @@ func (e *Echo) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, i //check if the file size is greater than max file size of the service //if yes we will return 200 ok or 204 no modification, it depends on the configuration of the service - if e.maxFileSize != 0 && e.maxFileSize < file.Len() { - status, file, httpMsgAfter := e.generalFunc.IfMaxFileSizeExc(e.returnOrigIfMaxSizeExc, e.serviceName, e.methodName, file, e.maxFileSize, utils.BlockPagePath, fileSize) + if e.maxFileSize != 0 && e.maxFileSize < len(file) { + status, file, httpMsgAfter := e.generalFunc.IfMaxFileSizeExc(e.returnOrigIfMaxSizeExc, e.serviceName, e.methodName, bytes.NewBuffer(file), e.maxFileSize, utils.BlockPagePath, fileSize) fileAfterPrep, httpMsgAfter := e.generalFunc.IfStatusIs204WithFile(e.methodName, status, file, isGzip, reqContentType, httpMsgAfter, true) if fileAfterPrep == nil && httpMsgAfter == nil { logging.Logger.Info(utils.PrepareLogMsg(e.xICAPMetadata, e.serviceName+" service has stopped processing")) @@ -103,7 +103,7 @@ func (e *Echo) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, i return status, nil, nil, msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs } - scannedFile := file.Bytes() + scannedFile := file //returning the scanned file if everything is ok scannedFile = e.generalFunc.PreparingFileAfterScanning(scannedFile, reqContentType, e.methodName) diff --git a/service/services/hashlocal/config.go b/service/services/hashlocal/config.go index f92ba089..0f07da83 100644 --- a/service/services/hashlocal/config.go +++ b/service/services/hashlocal/config.go @@ -35,6 +35,10 @@ type Hashlocal struct { CaseBlockHttpBody bool ExceptionPage string IcapHeaders textproto.MIMEHeader + serviceHeaders map[string]string + msgHeadersBeforeProcessing map[string]interface{} + msgHeadersAfterProcessing map[string]interface{} + vendorMsgs map[string]interface{} } func InitHashlocalConfig(serviceName string) { diff --git a/service/services/hashlocal/hashlocal.go b/service/services/hashlocal/hashlocal.go index 6884f9cd..1ff33ccb 100644 --- a/service/services/hashlocal/hashlocal.go +++ b/service/services/hashlocal/hashlocal.go @@ -6,75 +6,120 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/hex" - "fmt" utils "icapeg/consts" "icapeg/logging" + "icapeg/service/services-utilities/ContentTypes" "io" "log" "net/http" "net/textproto" "os" + "path/filepath" "strconv" "strings" "time" ) -// Processing is a func used for to processing the http message -func (h *Hashlocal) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, interface{}, map[string]string, map[string]interface{}, - map[string]interface{}, map[string]interface{}) { - serviceHeaders := make(map[string]string) - serviceHeaders["X-ICAP-Metadata"] = h.xICAPMetadata - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has started processing")) - msgHeadersBeforeProcessing := h.generalFunc.LogHTTPMsgHeaders(h.methodName) - msgHeadersAfterProcessing := make(map[string]interface{}) - vendorMsgs := make(map[string]interface{}) - h.IcapHeaders = IcapHeader - h.IcapHeaders.Add("X-ICAP-Metadata", h.xICAPMetadata) - // no need to scan part of the file, this service needs all the file at ine time +func (h *Hashlocal) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + h.initProcessing(IcapHeader) + if partial { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, - h.serviceName+" service has stopped processing partially")) - return utils.Continue, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs + return h.handlePartialProcessing() } - if h.methodName == utils.ICAPModeResp { - if h.httpMsg.Response != nil { - if h.httpMsg.Response.StatusCode == 206 { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing byte range received")) - return utils.NoModificationStatusCodeStr, h.httpMsg, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } - } + + file, reqContentType, err := h.extractFile() + if err != nil { + return h.handleError(utils.InternalServerErrStatusCodeStr, err) + } + + if h.methodName == utils.ICAPModeReq && h.httpMsg.Request.Method == http.MethodConnect { + return h.handleConnectMethod(file) } - isGzip := false + + fileName, contentType := h.getFileNameAndContentType() + fileSize := h.getFileSize(file) + ///////////////////////// ExceptionPagePath := utils.BlockPagePath + // fileExtension := h.generalFunc.GetMimeExtension(file, contentType[0], fileName) + var fileExtension string + filehead, err := h.httpMsg.StorageClient.ReadFileHeader(h.httpMsg.StorageKey) + if err != nil { + // Handle the error by extracting the file extension from the filename + logging.Logger.Warn(utils.PrepareLogMsg(h.xICAPMetadata, + "failed to read file header, falling back to file extension from filename: "+err.Error())) + fileExtension = filepath.Ext(fileName)[1:] + } else { + // Determine the file extension using the header data + fileExtension = h.generalFunc.GetMimeExtension(filehead, contentType[0], fileName) + } + h.FileHash, err = h.calculateFileHash(file) + if err != nil { + logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" calculateFileHash error : "+err.Error())) + } + ///////////////////////////////// + isProcess, _, _ := h.generalFunc.CheckTheExtension(fileExtension, h.extArrs, + h.processExts, h.rejectExts, h.bypassExts, h.return400IfFileExtRejected, h.isGzip(), + h.serviceName, h.methodName, h.FileHash, h.httpMsg.Request.RequestURI, reqContentType, bytes.NewBuffer(file), ExceptionPagePath, fmt.Sprint(fileSize)) - if h.ExceptionPage != "" { - ExceptionPagePath = h.ExceptionPage + + /////////////////////////////////// + + if h.isMaxFileSizeExceeded(fileSize) { + return h.handleMaxFileSizeExceeded(bytes.NewBuffer(file), fileSize, reqContentType) } - //extracting the file from http message - file, reqContentType, err := h.generalFunc.CopyingFileToTheBuffer(h.methodName) + if fileSize == 0 { + return h.handleEmptyFile(file) + } + isMal, err := h.sendFileToScan(h.FileHash, isProcess) + if err != nil { + return h.handleScanningError(err, file) + } + + if isMal { + return h.handleMaliciousFile(fileSize) + } + return h.handleSuccessfulScan(file, fileSize, reqContentType) +} + +func (h *Hashlocal) initProcessing(IcapHeader textproto.MIMEHeader) { + h.serviceHeaders = map[string]string{"X-ICAP-Metadata": h.xICAPMetadata} + h.msgHeadersBeforeProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) + h.msgHeadersAfterProcessing = make(map[string]interface{}) + h.vendorMsgs = make(map[string]interface{}) + h.IcapHeaders = IcapHeader + h.IcapHeaders.Add("X-ICAP-Metadata", h.xICAPMetadata) + logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has started processing")) +} + +func (h *Hashlocal) handlePartialProcessing() (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing partially")) + return utils.Continue, nil, nil, h.msgHeadersBeforeProcessing, h.msgHeadersAfterProcessing, h.vendorMsgs +} + +func (h *Hashlocal) extractFile() ([]byte, ContentTypes.ContentType, error) { + file, reqContentType, err := h.generalFunc.CopyingFileToTheBuffer(h.methodName) if err != nil { logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" error: "+err.Error())) logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return utils.InternalServerErrStatusCodeStr, nil, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs } + return file, reqContentType, err +} - //if the http method is Connect, return the request as it is because it has no body - if h.methodName == utils.ICAPModeReq { - if h.httpMsg.Request.Method == http.MethodConnect { - return utils.OkStatusCodeStr, h.generalFunc.ReturningHttpMessageWithFile(h.methodName, file.Bytes()), - serviceHeaders, msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } - } +func (h *Hashlocal) handleError(status int, err error) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" error: "+err.Error())) + logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) + return status, nil, h.serviceHeaders, h.msgHeadersBeforeProcessing, h.msgHeadersAfterProcessing, h.vendorMsgs +} - //getting the extension of the file - var contentType []string +func (h *Hashlocal) handleConnectMethod(file []byte) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + return utils.OkStatusCodeStr, h.generalFunc.ReturningHttpMessageWithFile(h.methodName, file), h.serviceHeaders, h.msgHeadersBeforeProcessing, h.msgHeadersAfterProcessing, h.vendorMsgs +} +func (h *Hashlocal) getFileNameAndContentType() (string, []string) { + var contentType []string var fileName string if h.methodName == utils.ICAPModeReq { contentType = h.httpMsg.Request.Header["Content-Type"] @@ -86,132 +131,149 @@ func (h *Hashlocal) Processing(partial bool, IcapHeader textproto.MIMEHeader) (i if len(contentType) == 0 { contentType = append(contentType, "") } - + if filepath.Ext(fileName) == "" { + fileName += "." + utils.Unknown + } + if filepath.Ext(fileName) == "." { + fileName += utils.Unknown + } logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" file name : "+fileName)) + return fileName, contentType +} - fileExtension := h.generalFunc.GetMimeExtension(file.Bytes(), contentType[0], fileName) - //check if the file extension is a bypass extension - //if yes we will not modify the file, and we will return 204 No modifications - - hash := sha256.New() - f := file - _, err = hash.Write(f.Bytes()) - if err != nil { - fmt.Println(err.Error()) - } - fileSize := fmt.Sprintf("%v", file.Len()) - fileHash := hex.EncodeToString(hash.Sum([]byte(nil))) - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" file hash : "+fileHash)) - - //check if the file extension is a bypass extension - //if yes we will not modify the file, and we will return 204 No modifications - isProcess, icapStatus, httpMsg := h.generalFunc.CheckTheExtension(fileExtension, h.extArrs, - h.processExts, h.rejectExts, h.bypassExts, h.return400IfFileExtRejected, isGzip, - h.serviceName, h.methodName, fileHash, h.httpMsg.Request.RequestURI, reqContentType, file, ExceptionPagePath, fileSize) - if !isProcess { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return icapStatus, httpMsg, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs +func (h *Hashlocal) getFileSize(file []byte) int { + if h.methodName == utils.ICAPModeResp { + size, _ := h.httpMsg.StorageClient.Size(h.httpMsg.StorageKey) + return int(size) } - //check if the file size is greater than max file size of the service - //if yes we will return 200 ok or 204 no modification, it depends on the configuration of the service - if h.maxFileSize != 0 && h.maxFileSize < file.Len() { - status, file, httpMsg := h.generalFunc.IfMaxFileSizeExc(h.returnOrigIfMaxSizeExc, h.serviceName, h.methodName, file, h.maxFileSize, ExceptionPagePath, fileSize) - fileAfterPrep, httpMsg := h.generalFunc.IfStatusIs204WithFile(h.methodName, status, file, isGzip, reqContentType, httpMsg, true) - if fileAfterPrep == nil && httpMsg == nil { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return utils.InternalServerErrStatusCodeStr, nil, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } - switch msg := httpMsg.(type) { - case *http.Request: - msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return status, msg, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - case *http.Response: - msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return status, msg, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return status, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs + return len(file) +} + +func (h *Hashlocal) isMaxFileSizeExceeded(fileSize int) bool { + return h.maxFileSize != 0 && h.maxFileSize < fileSize +} + +func (h *Hashlocal) handleMaxFileSizeExceeded(file *bytes.Buffer, fileSize int, reqContentType ContentTypes.ContentType) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + status, file, httpMsg := h.generalFunc.IfMaxFileSizeExc(h.returnOrigIfMaxSizeExc, h.serviceName, h.methodName, file, h.maxFileSize, h.ExceptionPagePath(), fmt.Sprint(fileSize)) + fileAfterPrep, httpMsg := h.generalFunc.IfStatusIs204WithFile(h.methodName, status, file, h.isGzip(), reqContentType, httpMsg, true) + if fileAfterPrep == nil && httpMsg == nil { + return h.handleError(utils.InternalServerErrStatusCodeStr, fmt.Errorf("fileAfterPrep bytes is null")) } + return h.prepareHttpResponse(status, httpMsg, fileAfterPrep) +} - scannedFile := file.Bytes() - isMal, err := h.sendFileToScan(file) - if err != nil && !h.BypassOnApiError { - logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" error: "+err.Error())) +func (h *Hashlocal) handleEmptyFile(file []byte) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing zero file length")) + return utils.NoModificationStatusCodeStr, h.generalFunc.ReturningHttpMessageWithFile(h.methodName, file), h.serviceHeaders, h.msgHeadersBeforeProcessing, h.msgHeadersAfterProcessing, h.vendorMsgs +} + +func (h *Hashlocal) handleScanningError(err error, file []byte) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" error: "+err.Error())) + if !h.BypassOnApiError { if strings.Contains(err.Error(), "context deadline exceeded") { - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return utils.RequestTimeOutStatusCodeStr, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs + return h.handleError(utils.RequestTimeOutStatusCodeStr, err) } - // its suppose to be InternalServerErrStatusCodeStr but need to be handled - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - return utils.BadRequestStatusCodeStr, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs + return h.handleError(utils.BadRequestStatusCodeStr, err) } + logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" BypassOnApiError true")) + return utils.NoModificationStatusCodeStr, h.generalFunc.ReturningHttpMessageWithFile(h.methodName, file), h.serviceHeaders, h.msgHeadersBeforeProcessing, h.msgHeadersAfterProcessing, h.vendorMsgs +} - if isMal { - logging.Logger.Debug(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+": file is not safe")) - if h.methodName == utils.ICAPModeResp { - - errPage := h.generalFunc.GenHtmlPage(ExceptionPagePath, utils.ErrPageReasonFileIsNotSafe, h.serviceName, h.FileHash, h.httpMsg.Request.RequestURI, fileSize, h.xICAPMetadata) - - h.httpMsg.Response = h.generalFunc.ErrPageResp(h.CaseBlockHttpResponseCode, errPage.Len()) - if h.CaseBlockHttpBody { - h.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(errPage.Bytes())) - } else { - var r []byte - h.httpMsg.Response.Body = io.NopCloser(bytes.NewBuffer(r)) - delete(h.httpMsg.Response.Header, "Content-Type") - delete(h.httpMsg.Response.Header, "Content-Length") - } - logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return utils.OkStatusCodeStr, h.httpMsg.Response, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } else { - htmlPage, req, err := h.generalFunc.ReqModErrPage(utils.ErrPageReasonFileIsNotSafe, h.serviceName, h.FileHash, fileSize) - if err != nil { - logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" error: "+err.Error())) - - return utils.InternalServerErrStatusCodeStr, nil, nil, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - } - req.Body = io.NopCloser(htmlPage) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - return utils.OkStatusCodeStr, req, serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs +func (h *Hashlocal) handleMaliciousFile(fileSize int) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + logging.Logger.Debug(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+": file is not safe")) + if h.methodName == utils.ICAPModeResp { + serviceHeadersStr := mapToString(h.serviceHeaders) + errPage := h.generalFunc.GenHtmlPage(h.ExceptionPagePath(), utils.ErrPageReasonFileIsNotSafe, h.serviceName, h.FileHash, h.httpMsg.Request.RequestURI, fmt.Sprint(fileSize), serviceHeadersStr) + h.httpMsg.Response = h.generalFunc.ErrPageResp(h.CaseBlockHttpResponseCode, errPage.Len()) + h.saveErrorPageBody(errPage.Bytes()) + return h.prepareHttpResponse(utils.OkStatusCodeStr, h.httpMsg.Response, nil) + } else { + htmlPage, req, err := h.generalFunc.ReqModErrPage(utils.ErrPageReasonFileIsNotSafe, h.serviceName, h.FileHash, fmt.Sprint(fileSize)) + if err != nil { + return h.handleError(utils.InternalServerErrStatusCodeStr, err) } + req.Body = io.NopCloser(htmlPage) + return h.prepareHttpResponse(utils.OkStatusCodeStr, req, nil) } +} - //returning the scanned file if everything is ok - //scannedFile = h.generalFunc.PreparingFileAfterScanning(scannedFile, reqContentType, h.methodName) +func (h *Hashlocal) handleSuccessfulScan(scannedFile []byte, fileSize int, reqContentType ContentTypes.ContentType) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + // Log headers and processing stop h.generalFunc.LogHTTPMsgHeaders(h.methodName) logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) - msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) - // OkStatusCodeStr => NoModificationStatusCodeStr - /* - return utils.NoModificationStatusCodeStr, - h.generalFunc.ReturningHttpMessageWithFile(h.methodName, scannedFile), serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs - /*return utils.NoModificationStatusCodeStr, nil, serviceHeaders, msgHeadersBeforeProcessing, - msgHeadersAfterProcessing, vendorMsgs */ - /*return utils.NoModificationStatusCodeStr, - h.ReturningHttpMessageWithFile(h.methodName, scannedFile, h.OriginalMsg), serviceHeaders, - msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs*/ + h.msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) + + // Process the scanned file scannedFile = h.generalFunc.PreparingFileAfterScanning(scannedFile, reqContentType, h.methodName) return utils.NoModificationStatusCodeStr, h.generalFunc.ReturningHttpMessageWithFile(h.methodName, scannedFile), - serviceHeaders, msgHeadersBeforeProcessing, msgHeadersAfterProcessing, vendorMsgs + h.serviceHeaders, h.msgHeadersBeforeProcessing, h.msgHeadersAfterProcessing, h.vendorMsgs +} + +func (h *Hashlocal) saveErrorPageBody(body []byte) { + if h.CaseBlockHttpBody { + h.httpMsg.StorageClient.Save(h.httpMsg.StorageKey, body) + } else { + var r []byte + h.httpMsg.StorageClient.Save(h.httpMsg.StorageKey, r) + delete(h.httpMsg.Response.Header, "Content-Type") + delete(h.httpMsg.Response.Header, "Content-Length") + } +} +func (h *Hashlocal) prepareHttpResponse(status int, httpMsg interface{}, fileAfterPrep []byte) (int, interface{}, map[string]string, map[string]interface{}, map[string]interface{}, map[string]interface{}) { + if httpMsg != nil { + switch msg := httpMsg.(type) { + case *http.Request: + if fileAfterPrep != nil { + msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) + } + case *http.Response: + if fileAfterPrep != nil { + msg.Body = io.NopCloser(bytes.NewBuffer(fileAfterPrep)) + } + } + } + h.msgHeadersAfterProcessing = h.generalFunc.LogHTTPMsgHeaders(h.methodName) + logging.Logger.Info(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" service has stopped processing")) + return status, httpMsg, h.serviceHeaders, h.msgHeadersBeforeProcessing, h.msgHeadersAfterProcessing, h.vendorMsgs +} + +func (h *Hashlocal) ExceptionPagePath() string { + if h.ExceptionPage != "" { + return h.ExceptionPage + } + return utils.BlockPagePath +} + +func (h *Hashlocal) isGzip() bool { + return false +} +// New function to calculate file hash without consuming memory +func (h *Hashlocal) calculateFileHash(file []byte) (string, error) { + var fileHash string + if h.methodName == utils.ICAPModeReq || h.httpMsg.StorageClient == nil { + + hash := sha256.New() + _, err := hash.Write(file) + if err != nil { + logging.Logger.Error(utils.PrepareLogMsg(h.xICAPMetadata, h.serviceName+" calculateFileHash error : "+err.Error())) + + } + fileHash = hex.EncodeToString(hash.Sum([]byte(nil))) + return fileHash, nil + } + + return h.httpMsg.StorageClient.ComputeHash(h.httpMsg.StorageKey) +} + +func mapToString(m map[string]string) string { + var sb strings.Builder + for key, value := range m { + sb.WriteString(fmt.Sprintf("\r\n")) + sb.WriteString(fmt.Sprintf("%s: %s", key, value)) + } + return sb.String() } // new functions @@ -242,7 +304,7 @@ func checkValueInFile(filePath, targetValue string) (bool, error) { return true, nil } } - + // Check for errors during scanning if err := scanner.Err(); err != nil { return false, err @@ -251,37 +313,42 @@ func checkValueInFile(filePath, targetValue string) (bool, error) { // If the value is not found in any line return false, nil } -func (h *Hashlocal) sendFileToScan(f *bytes.Buffer) (bool, error) { +func (h *Hashlocal) sendFileToScan(f string, isProcess bool) (bool, error) { + if !isProcess { + return false, nil + } + h.FileHash = f //hash code - hash := sha256.New() - _, _ = io.Copy(hash, f) - bs := hash.Sum(nil) - pass := hex.EncodeToString(bs[:]) - h.FileHash = pass + // hash := sha256.New() + // _, _ = io.Copy(hash, f) + // bs := hash.Sum(nil) + // pass := hex.EncodeToString(bs[:]) + // the file path filePath := "./hash_file/hash_file_path.txt" // Check if the target value is present in the file - found, err := checkValueInFile(filePath, pass) + found, err := checkValueInFile(filePath, h.FileHash) if err != nil { fmt.Printf("Error: %v\n", err) return false, nil } - logfile, err := os.OpenFile("log.json", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + logfile, err := os.OpenFile("logs/HashlocalLog.json", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { log.Fatal(err) } defer logfile.Close() log.SetOutput(logfile) if found { - log.Printf("Value '%s' found in the file.\n", pass) + log.Printf("Value '%s' found in the file.\n", h.FileHash) return true, nil } else { - log.Printf("Value '%s' not found in the file.\n", pass) + log.Printf("Value '%s' not found in the file.\n", h.FileHash) return false, nil } } + func (e *Hashlocal) ISTagValue() string { epochTime := strconv.FormatInt(time.Now().Unix(), 10) return "epoch-" + epochTime diff --git a/storage/auto.go b/storage/auto.go new file mode 100644 index 00000000..4776e8c4 --- /dev/null +++ b/storage/auto.go @@ -0,0 +1,298 @@ +package storage + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "hash" + "io" + "log" + "os" + "path/filepath" +) + +type AutoStorage struct { + memoryStorage *MemoryStorage + diskStorage *DiskStorage + maxMemorySize int64 +} + +func NewAutoStorage(maxMemorySize int64, diskPath string) *AutoStorage { + return &AutoStorage{ + memoryStorage: NewMemoryStorage(), + diskStorage: NewDiskStorage(diskPath), + maxMemorySize: maxMemorySize * 1024 * 1024, //To convert the memory size to MB to be compared with the file size + } +} + +func (as *AutoStorage) Save(key string, data []byte) error { + if int64(len(data)) > as.maxMemorySize { + return as.diskStorage.Save(key, data) + } + return as.memoryStorage.Save(key, data) +} + +func (as *AutoStorage) Load(key string) ([]byte, error) { + data, err := as.memoryStorage.Load(key) + if err == nil { + return data, nil + } + if errors.Is(err, os.ErrNotExist) { + return as.diskStorage.Load(key) + } + return nil, err +} + +func (as *AutoStorage) Delete(key string) error { + err := as.memoryStorage.Delete(key) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return as.diskStorage.Delete(key) +} + +func (as *AutoStorage) Size(key string) (int64, error) { + size, err := as.memoryStorage.Size(key) + if err == nil { + return size, nil + } + if errors.Is(err, os.ErrNotExist) { + return as.diskStorage.Size(key) + } + return 0, err +} + +func (as *AutoStorage) Append(key string, data []byte) error { + currentSize, err := as.Size(key) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return as.Save(key, data) + } + return err + } + + if currentSize+int64(len(data)) > as.maxMemorySize { + // Transition from memory storage to disk storage + if as.IsInMemory(key) { + + existingData, err := as.memoryStorage.Load(key) + if err != nil { + return err + } + + // Save existing data to disk + err = as.diskStorage.Save(key, existingData) + if err != nil { + return err + } + // Delete from memory storage + err = as.memoryStorage.Delete(key) + if err != nil { + return err + } + } + // Append the new data to disk storage + // Here, we ensure to append any remaining data after transitioning + err = as.diskStorage.Append(key, data) + if err != nil { + return err + } + + return nil + } + + // If the combined size is within the maxMemorySize limit, append to memory + return as.memoryStorage.Append(key, data) +} + +func (as *AutoStorage) AppendFromReader(key string, r io.Reader) error { + var buffer []byte + currentSize, err := as.Size(key) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + + // Read data from the io.Reader until the total read bytes exceed maxMemorySize + for { + chunk := make([]byte, 1024*1024) // 1MB buffer + n, err := r.Read(chunk) + if err != nil && err != io.EOF { + return err + } + if n > 0 { + buffer = append(buffer, chunk[:n]...) + currentSize += int64(n) + if currentSize > as.maxMemorySize { + break + } + } + if err == io.EOF { + break + } + } + + // Save to memory if within the maxMemorySize limit + if currentSize <= as.maxMemorySize { + return as.memoryStorage.Save(key, buffer) + } + + // Transition to disk storage + if buffer != nil { + // Save existing data to disk + err = as.diskStorage.Save(key, buffer) + if err != nil { + return err + } + + // Delete from memory storage + err = as.memoryStorage.Delete(key) + if err != nil { + return err + } + } + + // Continue reading the remaining data and appending to disk storage + _, err = io.Copy(&diskAppender{as.diskStorage, key}, r) + if err != nil { + return err + } + // Close the file after appending + return as.diskStorage.CloseFile(key) +} + +type diskAppender struct { + diskStorage *DiskStorage + key string +} + +func (da *diskAppender) Write(p []byte) (n int, err error) { + err = da.diskStorage.appendToOpenFile(da.key, p) + if err != nil { + return 0, err + } + return len(p), nil +} + +// Add the IsInMemory method +func (as *AutoStorage) IsInMemory(key string) bool { + _, err := as.memoryStorage.Load(key) + if err == nil { + return true + } + if errors.Is(err, os.ErrNotExist) { + return false + } + return false +} + +func (as *AutoStorage) GetBasePath() string { + return as.diskStorage.basePath +} + +// Add the ComputeHash method to calculate and return the file hash +func (as *AutoStorage) ComputeHash(key string) (string, error) { + isInMemory := as.IsInMemory(key) + + var hasher hash.Hash = sha256.New() + + if isInMemory { + data, err := as.memoryStorage.Load(key) + if err != nil { + return "", err + } + _, err = hasher.Write(data) + if err != nil { + return "", err + } + } else { + + path := filepath.Join(as.diskStorage.basePath, key) + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + _, err = io.Copy(hasher, file) + if err != nil { + return "", err + } + } + + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +// Add the Replace method to replace keyTo name to keyFrom then delete keyFrom +func (as *AutoStorage) ReplaceAndDelete(keyFrom, keyTo string) error { + isInMemory := as.IsInMemory(keyFrom) + if isInMemory { + data, err := as.memoryStorage.Load(keyFrom) + if err != nil { + return err + } + err = as.Save(keyTo, data) + if err != nil { + return err + } + as.memoryStorage.Delete(keyFrom) + return nil + } + // Check if newPath exists and delete it if it's a file + keyToPath := filepath.Join(as.diskStorage.basePath, keyTo) + _, err := os.Stat(keyToPath) + if err == nil { + if err := os.Remove(keyToPath); err != nil { + log.Println(err) + } + } + info, err := os.Stat(filepath.Join(as.diskStorage.basePath, keyFrom)) + if err != nil { + return err + } + oldPath := filepath.Join(as.diskStorage.basePath, keyFrom) + newPath := filepath.Join(as.diskStorage.basePath, keyTo) + as.diskStorage.mutex.RLock() + defer as.diskStorage.mutex.RUnlock() + as.diskStorage.sizes[keyTo] = info.Size() + return os.Rename(oldPath, newPath) + +} + +// ReadFileHeader reads the first 262 bytes of the file identified by the given key. +// If the file is smaller than 262 bytes, it reads the entire file. +func (as *AutoStorage) ReadFileHeader(key string) ([]byte, error) { + const maxHeaderSize = 262 // Maximum file header size to read + + var header []byte + var err error + + if as.IsInMemory(key) { + data, err := as.memoryStorage.Load(key) + if err != nil { + return nil, err + } + // Determine the number of bytes to read (min of 262 or data length) + readLen := len(data) + if readLen > maxHeaderSize { + readLen = maxHeaderSize + } + header = data[:readLen] + } else { + path := filepath.Join(as.diskStorage.basePath, key) + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + // Allocate a buffer of 262 bytes + buffer := make([]byte, maxHeaderSize) + n, err := file.Read(buffer) + if err != nil && err != io.EOF { + return nil, err + } + header = buffer[:n] + } + + return header, err +} diff --git a/storage/disk.go b/storage/disk.go new file mode 100644 index 00000000..8e00a9c5 --- /dev/null +++ b/storage/disk.go @@ -0,0 +1,157 @@ +package storage + +import ( + "io" + "os" + "path/filepath" + "sync" +) + +type DiskStorage struct { + basePath string + sizes map[string]int64 + mutex sync.RWMutex + openFile map[string]*os.File +} + +func NewDiskStorage(basePath string) *DiskStorage { + return &DiskStorage{ + basePath: basePath, + sizes: make(map[string]int64), + openFile: make(map[string]*os.File), + } +} + +func (ds *DiskStorage) Save(key string, data []byte) error { + path := filepath.Join(ds.basePath, key) + err := os.WriteFile(path, data, 0644) + if err == nil { + ds.mutex.Lock() + ds.sizes[key] = int64(len(data)) + ds.mutex.Unlock() + } + return err +} + +func (ds *DiskStorage) Load(key string) ([]byte, error) { + path := filepath.Join(ds.basePath, key) + return os.ReadFile(path) +} + +func (ds *DiskStorage) Delete(key string) error { + path := filepath.Join(ds.basePath, key) + err := os.Remove(path) + if err == nil { + ds.mutex.Lock() + delete(ds.sizes, key) + ds.mutex.Unlock() + } + return err +} + +func (ds *DiskStorage) Size(key string) (int64, error) { + ds.mutex.RLock() + defer ds.mutex.RUnlock() + size, exists := ds.sizes[key] + if !exists { + return 0, os.ErrNotExist + } + return size, nil +} + +func (ds *DiskStorage) Append(key string, data []byte) error { + path := filepath.Join(ds.basePath, key) + + // Check if the file exists + if _, err := os.Stat(path); os.IsNotExist(err) { + // If the file does not exist, create it and save the data + return ds.Save(key, data) + } + + // If the file exists, append the data + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + _, err = file.Write(data) + if err == nil { + ds.mutex.Lock() + ds.sizes[key] += int64(len(data)) + ds.mutex.Unlock() + } + return err +} + +func (ds *DiskStorage) AppendFromReader(key string, r io.Reader) error { + path := filepath.Join(ds.basePath, key) + + // Check if the file exists + if _, err := os.Stat(path); os.IsNotExist(err) { + // If the file does not exist, create it + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + } + + // Open the file in append mode + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + // Append data from the reader + written, err := io.Copy(file, r) + if err == nil { + ds.mutex.Lock() + ds.sizes[key] += written + ds.mutex.Unlock() + } + return err +} +func (ds *DiskStorage) appendToOpenFile(key string, data []byte) error { + ds.mutex.Lock() + defer ds.mutex.Unlock() + + path := filepath.Join(ds.basePath, key) + + // Check if the file is already open + file, exists := ds.openFile[key] + if !exists { + // Open the file in append mode + var err error + file, err = os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + ds.openFile[key] = file + } + + _, err := file.Write(data) + if err == nil { + ds.sizes[key] += int64(len(data)) + } + return err +} + +func (ds *DiskStorage) CloseFile(key string) error { + ds.mutex.Lock() + defer ds.mutex.Unlock() + + file, exists := ds.openFile[key] + if !exists { + return nil + } + + err := file.Close() + if err != nil { + return err + } + + delete(ds.openFile, key) + return nil +} diff --git a/storage/memory.go b/storage/memory.go new file mode 100644 index 00000000..71bb779b --- /dev/null +++ b/storage/memory.go @@ -0,0 +1,90 @@ +package storage + +import ( + "bytes" + "io" + "os" + "sync" +) + +type MemoryStorage struct { + storage map[string][]byte + sizes map[string]int64 + mutex sync.RWMutex +} + +func NewMemoryStorage() *MemoryStorage { + return &MemoryStorage{ + storage: make(map[string][]byte), + sizes: make(map[string]int64), + } +} + +func (ms *MemoryStorage) Save(key string, data []byte) error { + ms.mutex.Lock() + defer ms.mutex.Unlock() + ms.storage[key] = data + ms.sizes[key] = int64(len(data)) + return nil +} + +func (ms *MemoryStorage) Load(key string) ([]byte, error) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + data, exists := ms.storage[key] + if !exists { + return nil, os.ErrNotExist + } + return data, nil +} + +func (ms *MemoryStorage) Delete(key string) error { + ms.mutex.Lock() + defer ms.mutex.Unlock() + delete(ms.storage, key) + delete(ms.sizes, key) + return nil +} + +func (ms *MemoryStorage) Size(key string) (int64, error) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + size, exists := ms.sizes[key] + if !exists { + return 0, os.ErrNotExist + } + return size, nil +} + +func (ms *MemoryStorage) Append(key string, data []byte) error { + ms.mutex.Lock() + defer ms.mutex.Unlock() + _, exists := ms.storage[key] + if !exists { + ms.storage[key] = data + ms.sizes[key] = int64(len(data)) + return nil + } + ms.storage[key] = append(ms.storage[key], data...) + ms.sizes[key] += int64(len(data)) + return nil +} + +func (ms *MemoryStorage) AppendFromReader(key string, r io.Reader) error { + ms.mutex.Lock() + defer ms.mutex.Unlock() + var buffer bytes.Buffer + written, err := io.Copy(&buffer, r) + if err != nil { + return err + } + _, exists := ms.storage[key] + if !exists { + ms.storage[key] = buffer.Bytes() + ms.sizes[key] = written + return nil + } + ms.storage[key] = append(ms.storage[key], buffer.Bytes()...) + ms.sizes[key] += written + return nil +} diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 00000000..862a2704 --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,17 @@ +package storage + +import "io" + +type StorageClient interface { + Save(key string, data []byte) error + Load(key string) ([]byte, error) + Delete(key string) error + Size(key string) (int64, error) + Append(key string, data []byte) error + AppendFromReader(key string, r io.Reader) error + IsInMemory(key string) bool + GetBasePath() string + ComputeHash(key string) (string, error) + ReplaceAndDelete(keyFrom, keyTo string) error + ReadFileHeader(key string) ([]byte, error) +} diff --git a/temp/exception-page.html b/temp/exception-page.html index e46163b2..fee2b028 100644 --- a/temp/exception-page.html +++ b/temp/exception-page.html @@ -164,7 +164,7 @@

} else if (res == "fileRejected") { msg.innerText = "Access denied ! file type not allowed " } else if (res == "fileIsNotSafe") { - msg.innerText = "Access denied ! file contains various" + msg.innerText = "Access denied ! file contains Virus" }else if (res == "fileIsNotFound") { msg.innerText = "Access denied ! file not found" }else if (res == "partialContent") { From f9f01da26b3b272cb82332ee1eeb96039c07e731 Mon Sep 17 00:00:00 2001 From: Elbahkiry Date: Wed, 4 Sep 2024 10:59:17 +0300 Subject: [PATCH 3/6] fix echo service --- service/services/echo/echo.go | 58 +++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/service/services/echo/echo.go b/service/services/echo/echo.go index 8b5de0de..5b952059 100644 --- a/service/services/echo/echo.go +++ b/service/services/echo/echo.go @@ -5,6 +5,7 @@ import ( "fmt" utils "icapeg/consts" "icapeg/logging" + "icapeg/service/services-utilities/ContentTypes" "io" "net/http" "net/textproto" @@ -31,7 +32,7 @@ func (e *Echo) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, i isGzip := false //extracting the file from http message - file, reqContentType, err := e.generalFunc.CopyingFileToTheBuffer(e.methodName) + file, reqContentType, err := e.CopyingFileToTheBuffer(e.methodName) if err != nil { logging.Logger.Error(utils.PrepareLogMsg(e.xICAPMetadata, e.serviceName+" error: "+err.Error())) logging.Logger.Info(utils.PrepareLogMsg(e.xICAPMetadata, e.serviceName+" service has stopped processing")) @@ -66,19 +67,19 @@ func (e *Echo) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, i //check if the file extension is a bypass extension //if yes we will not modify the file, and we will return 204 No modifications - isProcess, icapStatus, httpMsg := e.generalFunc.CheckTheExtension(fileExtension, e.extArrs, + isProcess, _, _ := e.generalFunc.CheckTheExtension(fileExtension, e.extArrs, e.processExts, e.rejectExts, e.bypassExts, e.return400IfFileExtRejected, isGzip, e.serviceName, e.methodName, EchoIdentifier, e.httpMsg.Request.RequestURI, reqContentType, bytes.NewBuffer(file), utils.BlockPagePath, fileSize) - if !isProcess { - logging.Logger.Info(utils.PrepareLogMsg(e.xICAPMetadata, e.serviceName+" service has stopped processing")) - msgHeadersAfterProcessing = e.generalFunc.LogHTTPMsgHeaders(e.methodName) - return icapStatus, httpMsg, serviceHeaders, msgHeadersBeforeProcessing, - msgHeadersAfterProcessing, vendorMsgs - } + // if !isProcess { + // logging.Logger.Info(utils.PrepareLogMsg(e.xICAPMetadata, e.serviceName+" service has stopped processing")) + // msgHeadersAfterProcessing = e.generalFunc.LogHTTPMsgHeaders(e.methodName) + // return icapStatus, httpMsg, serviceHeaders, msgHeadersBeforeProcessing, + // msgHeadersAfterProcessing, vendorMsgs + // } //check if the file size is greater than max file size of the service //if yes we will return 200 ok or 204 no modification, it depends on the configuration of the service - if e.maxFileSize != 0 && e.maxFileSize < len(file) { + if e.maxFileSize != 0 && e.maxFileSize < len(file) && isProcess { status, file, httpMsgAfter := e.generalFunc.IfMaxFileSizeExc(e.returnOrigIfMaxSizeExc, e.serviceName, e.methodName, bytes.NewBuffer(file), e.maxFileSize, utils.BlockPagePath, fileSize) fileAfterPrep, httpMsgAfter := e.generalFunc.IfStatusIs204WithFile(e.methodName, status, file, isGzip, reqContentType, httpMsgAfter, true) if fileAfterPrep == nil && httpMsgAfter == nil { @@ -117,3 +118,42 @@ func (e *Echo) ISTagValue() string { epochTime := strconv.FormatInt(time.Now().Unix(), 10) return "epoch-" + epochTime } + +// CopyingFileToTheBuffer is a func which used for extracting a file from the body of the http message +func (e *Echo) CopyingFileToTheBuffer(methodName string) ([]byte, ContentTypes.ContentType, error) { + logging.Logger.Info(utils.PrepareLogMsg(e.xICAPMetadata, "extracting the body of HTTP message")) + var file []byte + var err error + var reqContentType ContentTypes.ContentType + reqContentType = nil + switch methodName { + case utils.ICAPModeReq: + file, reqContentType, err = e.copyingFileToTheBufferReq() + case utils.ICAPModeResp: + file, err = e.copyingFileToTheBufferResp() + } + if err != nil { + return nil, nil, err + } + return file, reqContentType, nil +} + +// copyingFileToTheBufferResp is a utility function for CopyingFileToTheBuffer func +// it's used for extracting a file from the body of the http response +func (e *Echo) copyingFileToTheBufferResp() ([]byte, error) { + file, err := e.httpMsg.StorageClient.Load(e.httpMsg.StorageKey) + if err != nil { + return file, err + } + return file, nil +} + +// copyingFileToTheBufferReq is a utility function for CopyingFileToTheBuffer func +// it's used for extracting a file from the body of the http request +func (e *Echo) copyingFileToTheBufferReq() ([]byte, ContentTypes.ContentType, error) { + reqContentType := ContentTypes.GetContentType(e.httpMsg.Request) + // getting the file from request and store it in buf as a type of bytes.Buffer + file := reqContentType.GetFileFromRequest() + return file.Bytes(), reqContentType, nil + +} From 6d62feb655c5861adfa8325e3ccf6c65339c81b8 Mon Sep 17 00:00:00 2001 From: Elbahkiry Date: Thu, 5 Sep 2024 12:22:31 +0300 Subject: [PATCH 4/6] fix pipeline problem --- .../services-utilities/general-functions/general-functions.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/services-utilities/general-functions/general-functions.go b/service/services-utilities/general-functions/general-functions.go index a267bee6..92da0d11 100644 --- a/service/services-utilities/general-functions/general-functions.go +++ b/service/services-utilities/general-functions/general-functions.go @@ -261,6 +261,8 @@ func (f *GeneralFunc) IfMaxFileSizeExc(returnOrigIfMaxSizeExc bool, serviceName, htmlErrPage := f.GenHtmlPage(BlockPagePath, utils.ErrPageReasonMaxFileExceeded, serviceName, "-", f.httpMsg.Request.RequestURI, fileSize, f.xICAPMetadata) f.httpMsg.Response = f.ErrPageResp(http.StatusForbidden, htmlErrPage.Len()) + // Save the block page to the response + f.httpMsg.StorageClient.Save(f.httpMsg.StorageKey, htmlErrPage.Bytes()) return utils.OkStatusCodeStr, htmlErrPage, f.httpMsg.Response } else { htmlPage, req, err := f.ReqModErrPage(utils.ErrPageReasonMaxFileExceeded, serviceName, "-", fileSize) From 23e115e8676aefb22a1e5e5324a29c09063d78b8 Mon Sep 17 00:00:00 2001 From: Elbahkiry Date: Thu, 5 Sep 2024 15:43:11 +0300 Subject: [PATCH 5/6] fix pipeline problem related to request mode --- api/icap-request.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/icap-request.go b/api/icap-request.go index 72ab6037..ae4af960 100644 --- a/api/icap-request.go +++ b/api/icap-request.go @@ -223,7 +223,9 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) { //icap.Request.Response IcapStatusCode, httpMsg, serviceHeaders, httpMshHeadersBeforeProcessing, httpMshHeadersAfterProcessing, vendorMsgs := requiredService.Processing(partial, i.req.Header) - defer i.req.StorageClient.Delete(i.req.StorageKey) // Delete the file in the tmp directory after processing(useful when error happens) + if i.methodName == utils.ICAPModeResp { + defer i.req.StorageClient.Delete(i.req.StorageKey) // Delete the file in the tmp directory after processing(useful when error happens) + } // adding the headers which the service wants to add them in the ICAP response logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, From dd2d3c839bfcd71cf77964b9d973c3dd8ebc2087 Mon Sep 17 00:00:00 2001 From: Elbahkiry Date: Thu, 5 Sep 2024 16:42:42 +0300 Subject: [PATCH 6/6] fix pipeline problem related to request mode --- service/services/echo/echo.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/service/services/echo/echo.go b/service/services/echo/echo.go index 5b952059..85836acf 100644 --- a/service/services/echo/echo.go +++ b/service/services/echo/echo.go @@ -67,15 +67,15 @@ func (e *Echo) Processing(partial bool, IcapHeader textproto.MIMEHeader) (int, i //check if the file extension is a bypass extension //if yes we will not modify the file, and we will return 204 No modifications - isProcess, _, _ := e.generalFunc.CheckTheExtension(fileExtension, e.extArrs, + isProcess, icapStatus, httpMsg := e.generalFunc.CheckTheExtension(fileExtension, e.extArrs, e.processExts, e.rejectExts, e.bypassExts, e.return400IfFileExtRejected, isGzip, e.serviceName, e.methodName, EchoIdentifier, e.httpMsg.Request.RequestURI, reqContentType, bytes.NewBuffer(file), utils.BlockPagePath, fileSize) - // if !isProcess { - // logging.Logger.Info(utils.PrepareLogMsg(e.xICAPMetadata, e.serviceName+" service has stopped processing")) - // msgHeadersAfterProcessing = e.generalFunc.LogHTTPMsgHeaders(e.methodName) - // return icapStatus, httpMsg, serviceHeaders, msgHeadersBeforeProcessing, - // msgHeadersAfterProcessing, vendorMsgs - // } + if !isProcess && e.methodName == utils.ICAPModeReq { + logging.Logger.Info(utils.PrepareLogMsg(e.xICAPMetadata, e.serviceName+" service has stopped processing")) + msgHeadersAfterProcessing = e.generalFunc.LogHTTPMsgHeaders(e.methodName) + return icapStatus, httpMsg, serviceHeaders, msgHeadersBeforeProcessing, + msgHeadersAfterProcessing, vendorMsgs + } //check if the file size is greater than max file size of the service //if yes we will return 200 ok or 204 no modification, it depends on the configuration of the service