Merge pull request #169 from Elbahkiry/disk_storage

add disk storage

README.md

@@ -98,13 +98,13 @@ You should see something like, ```ICAP server is running on localhost:1344 ...``
 - **echo**
 > simply it's like when you bing url,so what we send will be received 
 
-- ## clamav**
+- ## **clamav**
 > ClamAV is an open source (GPLv2) anti-virus toolkit, designed especially for e-mail scanning on mail gateways. It provides a number of utilities including a flexible and scalable multi-threaded daemon also helps to scan file quickly.
 
-- ## clhashlookup**
-> simply it helps to scan each file we need to check before we send to Api.
+- ## **clhashlookup**
+> simply it helps to scan each file we need to check before we send to API.
 
-- ## hashlocal**
+- ## **hashlocal**
 >  we will pretend that you want to download a file. Still, you don't know if it is safe or not, so     using this service helps you calculate the hash value of any file you download through the icapeg, then it checks if this hash value is available or not in hash_file if it is in the hash_file returns back an exception page, so if you tried to downloadany file of eicar test vrius it will appear an exception page,cause the hash file of thosefile is foun in our hash_file.
 
 
@@ -131,6 +131,8 @@ You should see something like, ```ICAP server is running on localhost:1344 ...``
         port = 1344
         services= ["echo", "virustotal", "clamav", "cloudmersive"]
         debugging_headers=true
+        max_file_size_on_memory = 10 # in megabytes
+        write_path_on_disk = "./tmp"
         ```
 
         - **port**
@@ -153,7 +155,13 @@ You should see something like, ```ICAP server is running on localhost:1344 ...``
           - **false**: Debugging headers should not be displayed with ICAP headers.
 
           - Any port number that isn't used in your machine.
+        - **max_file_size_on_memory** (MB)
 
+          The maximum file size in megabytes that will be handled in memory. Files exceeding this size will be stored and processed on disk. The default value is `10` MB.
+
+        - **write_path_on_disk**
+
+          The directory path where files exceeding `max_file_size_on_memory` will be stored on disk. The default path is `"./tmp"`.
       ## echo
 
       - **[echo] section** 
@@ -407,7 +415,7 @@ You should see something like, ```ICAP server is running on localhost:1344 ...``
         http_exception_has_body = true
         exception_page = "./temp/exception-page.html" # Location of the exception page for this service
         ```
-      - ### **New used variables **
+      - ### **New used variables**
 
        - **socket_path**
        It is a string variable it helps sending the HTTP msg body to the ClamAV through antivirus socket.

api/icap-request.go

@@ -11,7 +11,7 @@ import (
 	"icapeg/logging"
 	"icapeg/service"
 	"io"
-	"io/ioutil"
+	"log"
 	"math/rand"
 	"net/http"
 	"strconv"
@@ -63,7 +63,7 @@ func (i *ICAPRequest) RequestInitialization() (string, error) {
 	logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "checking if the service doesn't exist in toml file"))
 	i.serviceName = i.req.URL.Path[1:len(i.req.URL.Path)]
 	if !i.isServiceExists(xICAPMetadata) {
-		i.w.WriteHeader(utils.ICAPServiceNotFoundCodeStr, nil, false)
+		i.w.WriteHeader(utils.ICAPServiceNotFoundCodeStr, nil, false, i.req)
 		err := errors.New("service doesn't exist")
 		logging.Logger.Error(err.Error())
 		return xICAPMetadata, err
@@ -74,7 +74,7 @@ func (i *ICAPRequest) RequestInitialization() (string, error) {
 	i.methodName = i.req.Method
 	if i.methodName != "options" {
 		if !i.isMethodAllowed(xICAPMetadata) {
-			i.w.WriteHeader(utils.MethodNotAllowedForServiceCodeStr, nil, false)
+			i.w.WriteHeader(utils.MethodNotAllowedForServiceCodeStr, nil, false, i.req)
 			err := errors.New("method is not allowed")
 			logging.Logger.Error(err.Error())
 			return xICAPMetadata, err
@@ -87,11 +87,11 @@ func (i *ICAPRequest) RequestInitialization() (string, error) {
 
 	//adding important headers to options ICAP response
 	requiredService := service.GetService(i.vendor, i.serviceName, i.methodName,
-		&http_message.HttpMsg{Request: i.req.Request, Response: i.req.Response}, xICAPMetadata)
+		&http_message.HttpMsg{Request: i.req.Request, Response: i.req.Response, StorageClient: i.req.StorageClient, StorageKey: i.req.StorageKey}, xICAPMetadata)
 	logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "adding ISTAG Service Headers"))
 	i.addingISTAGServiceHeaders(requiredService.ISTagValue())
 
-	logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "checking if returning 24 to ICAP client is allowed or not"))
+	logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata, "checking if returning 204 to ICAP client is allowed or not"))
 	i.Is204Allowed = i.is204Allowed(xICAPMetadata)
 
 	i.isShadowServiceEnabled = config.AppCfg.ServicesInstances[i.serviceName].ShadowService
@@ -121,15 +121,18 @@ func (i *ICAPRequest) RequestProcessing(xICAPMetadata string) {
 		"processing ICAP request upon the service and method required"))
 	partial := false
 	if i.methodName != utils.ICAPModeOptions {
-		file := &bytes.Buffer{}
-		fileLen := 0
+		var fileLen int64 = 0
 
 		if i.methodName == utils.ICAPModeResp {
-			io.Copy(file, i.req.Response.Body)
-			fileLen = file.Len()
-			i.req.Response.Header.Set(utils.ContentLength, strconv.Itoa(len(file.Bytes())))
-			i.req.Response.Body = io.NopCloser(bytes.NewBuffer(file.Bytes()))
+			fileStorLen, err := i.req.StorageClient.Size(i.req.StorageKey)
+			if err != nil {
+				log.Println(err)
+				return
+			}
+			fileLen = fileStorLen
 
+			i.req.Response.Header.Set(utils.ContentLength, strconv.FormatInt(fileLen, 10))
+			//i.req.Response.Body = io.NopCloser(bytes.NewBuffer(file.Bytes()))
 		} else {
 			if i.req.Method == utils.ICAPModeReq {
 
@@ -140,7 +143,7 @@ func (i *ICAPRequest) RequestProcessing(xICAPMetadata string) {
 				} else {
 					i.req.OrgRequest = new
 				}
-				body, _ := ioutil.ReadAll(i.req.Request.Body)
+				body, _ := io.ReadAll(i.req.Request.Body)
 				i.req.OrgRequest.Body = io.NopCloser(bytes.NewBuffer(body))
 				i.req.OrgRequest.Header = i.req.Request.Header
 				i.req.OrgRequest.Header.Set(utils.ContentLength, strconv.Itoa(len(body)))
@@ -201,13 +204,6 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) {
 		defer i.req.Request.Body.Close()
 		defer i.req.OrgRequest.Body.Close()
 
-	} else {
-		defer i.req.Response.Body.Close()
-		//someString := "hello world nand hello go and more"
-		//r := strings.NewReader(someString)
-
-		//defer Original_rsp.Body.Close()
-
 	}
 	if i.req.Request == nil {
 		i.req.Request = &http.Request{}
@@ -216,7 +212,7 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) {
 	logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
 		"initialize the service by creating instance from the required service"))
 	requiredService := service.GetService(i.vendor, i.serviceName, i.methodName,
-		&http_message.HttpMsg{Request: i.req.Request, Response: i.req.Response}, xICAPMetadata)
+		&http_message.HttpMsg{Request: i.req.Request, Response: i.req.Response, StorageClient: i.req.StorageClient, StorageKey: i.req.StorageKey}, xICAPMetadata)
 
 	logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
 		"calling Processing func to process the http message which encapsulated inside the ICAP request"))
@@ -227,6 +223,9 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) {
 	//icap.Request.Response
 	IcapStatusCode, httpMsg, serviceHeaders, httpMshHeadersBeforeProcessing, httpMshHeadersAfterProcessing,
 		vendorMsgs := requiredService.Processing(partial, i.req.Header)
+	if i.methodName == utils.ICAPModeResp {
+		defer i.req.StorageClient.Delete(i.req.StorageKey) // Delete the file in the tmp directory after processing(useful when error happens)
+	}
 
 	// adding the headers which the service wants to add them in the ICAP response
 	logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
@@ -252,7 +251,7 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) {
 	case utils.InternalServerErrStatusCodeStr:
 		logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
 			i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.InternalServerErrStatusCodeStr)))
-		i.w.WriteHeader(IcapStatusCode, nil, false)
+		i.w.WriteHeader(IcapStatusCode, nil, false, i.req)
 	case utils.Continue:
 		logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
 			i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.Continue)))
@@ -264,45 +263,46 @@ func (i *ICAPRequest) RespAndReqMods(partial bool, xICAPMetadata string) {
 			i.req.Request.Body = io.NopCloser(bytes.NewBuffer(httpMsgBody.Bytes()))
 			i.req.OrgRequest.Body = io.NopCloser(bytes.NewBuffer(httpMsgBody.Bytes()))
 		} else {
-			i.req.Response.Body = io.NopCloser(bytes.NewBuffer(httpMsgBody.Bytes()))
+			// i.req.Response.Body = io.NopCloser(bytes.NewBuffer(httpMsgBody.Bytes()))
+			// i.req.StorageClient.Save(i.req.StorageKey, httpMsgBody.Bytes())
 		}
 		i.allHeaders(IcapStatusCode, httpMshHeadersBeforeProcessing, httpMshHeadersAfterProcessing, vendorMsgs,
 			xICAPMetadata)
 		i.RespAndReqMods(false, xICAPMetadata)
 	case utils.RequestTimeOutStatusCodeStr:
 		logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
 			i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.RequestTimeOutStatusCodeStr)))
-		i.w.WriteHeader(IcapStatusCode, nil, false)
+		i.w.WriteHeader(IcapStatusCode, nil, false, i.req)
 	case utils.NoModificationStatusCodeStr:
 		logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
 			i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.NoModificationStatusCodeStr)))
 		if i.Is204Allowed {
-			i.w.WriteHeader(utils.NoModificationStatusCodeStr, nil, false)
+			i.w.WriteHeader(utils.NoModificationStatusCodeStr, nil, false, i.req)
 		} else {
 
 			IcapStatusCode = utils.OkStatusCodeStr
 			if i.methodName == utils.ICAPModeReq {
 				IcapStatusCode = utils.OkStatusCodeStr
-				body, _ := ioutil.ReadAll(i.req.OrgRequest.Body)
+				body, _ := io.ReadAll(i.req.OrgRequest.Body)
 				i.req.Request.Body = io.NopCloser(bytes.NewBuffer(body))
 				i.req.Request.Header.Set(utils.ContentLength, strconv.Itoa(len(body)))
 				defer i.req.Request.Body.Close()
-				i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Request, true)
+				i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Request, true, i.req)
 			} else {
 				IcapStatusCode = utils.OkStatusCodeStr
-				i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true)
+				i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true, i.req)
 			}
 
 			//i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true)
 		}
 	case utils.OkStatusCodeStr:
 		logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
 			i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.OkStatusCodeStr)))
-		i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true)
+		i.w.WriteHeader(utils.OkStatusCodeStr, httpMsg, true, i.req)
 	case utils.BadRequestStatusCodeStr:
 		logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
 			i.serviceName+" returned ICAP response with status code "+strconv.Itoa(utils.BadRequestStatusCodeStr)))
-		i.w.WriteHeader(IcapStatusCode, httpMsg, true)
+		i.w.WriteHeader(IcapStatusCode, httpMsg, true, i.req)
 	}
 	i.allHeaders(IcapStatusCode, httpMshHeadersBeforeProcessing, httpMshHeadersAfterProcessing, vendorMsgs, xICAPMetadata)
 }
@@ -434,18 +434,18 @@ func (i *ICAPRequest) shadowService(xICAPMetadata string) {
 		i.h["X-ICAPeg-Shadow-Service"] = []string{"true"}
 	}
 	if i.Is204Allowed { // following RFC3507, if the request has Allow: 204 header, it is to be checked and if it doesn't exists, return the request as it is to the ICAP client, https://tools.ietf.org/html/rfc3507#section-4.6
-		i.w.WriteHeader(utils.NoModificationStatusCodeStr, nil, false)
+		i.w.WriteHeader(utils.NoModificationStatusCodeStr, nil, false, i.req)
 	} else {
 		if i.req.Method == "REQMOD" {
-			i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Request, true)
-			tempBody, _ := ioutil.ReadAll(i.req.Request.Body)
+			i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Request, true, i.req)
+			tempBody, _ := io.ReadAll(i.req.Request.Body)
 			i.w.Write(tempBody)
 			i.req.Request.Body = io.NopCloser(bytes.NewBuffer(tempBody))
 		} else if i.req.Method == "RESPMOD" {
-			i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Response, true)
-			tempBody, _ := ioutil.ReadAll(i.req.Response.Body)
+			i.w.WriteHeader(utils.OkStatusCodeStr, i.req.Response, true, i.req)
+			tempBody, _ := i.req.StorageClient.Load(i.req.StorageKey)
 			i.w.Write(tempBody)
-			i.req.Response.Body = io.NopCloser(bytes.NewBuffer(tempBody))
+			// i.req.Response.Body = io.NopCloser(bytes.NewBuffer(tempBody))
 		}
 	}
 }
@@ -486,7 +486,7 @@ func (i *ICAPRequest) optionsMode(serviceName, xICAPMetadata string) {
 		}
 	}
 	i.h.Set("Transfer-Preview", utils.Any)
-	i.w.WriteHeader(http.StatusOK, nil, false)
+	i.w.WriteHeader(http.StatusOK, nil, false, i.req)
 	i.optionsRespHeaders = i.LogICAPResHeaders(http.StatusOK)
 }
 
@@ -496,10 +496,19 @@ func (i *ICAPRequest) preview(xICAPMetadata string) *bytes.Buffer {
 	logging.Logger.Debug(utils.PrepareLogMsg(xICAPMetadata,
 		"getting the rest of the body from client after the service returned ICAP "+
 			"response with status code"+strconv.Itoa(utils.Continue)))
-	r := icap.GetTheRest()
-	c := io.NopCloser(r)
+	reader := i.req.GetTheRest()
+	readCloser := io.NopCloser(reader)
+	defer readCloser.Close()
+	if i.req.Method == utils.ICAPModeResp {
+		err := i.req.StorageClient.AppendFromReader(i.req.StorageKey, readCloser)
+		if err != nil {
+			log.Println(err)
+		}
+		return nil
+	}
+
 	buf := new(bytes.Buffer)
-	buf.ReadFrom(c)
+	buf.ReadFrom(readCloser)
 	return buf
 }
 

block-page.html

@@ -127,6 +127,7 @@ <h1>{{.Reason}}</h1>
 <div id="Details"  style="display:none;" class="error-info" >
     <p><strong>Requested URL: </strong>{{.RequestedURL}}</p>
     <p><strong>Service Name: </strong>{{.ServiceName}}</p>
+    <p><strong>File size: </strong>{{.Size}}</p>
     <p><strong>File Hash: </strong>{{.IdentifierId}}</p>
 </div>
 

config.toml

@@ -25,6 +25,8 @@ services= ["echo", "clhashlookup", "clamav","hashlocal"]
 debugging_headers=true
 web_server_host = "$_WEB_SERVER_HOST" #Example: "localhost:8081" , replace localhost with the ICAP server IP address.
 web_server_endpoint = "/service/message"  
+max_file_size_on_memory = 1  # megabytes
+write_path_on_disk="./tmp"
 
 [echo]
 vendor = "echo"

config/config.go

@@ -22,16 +22,18 @@ type serviceIcapInfo struct {
 
 // AppConfig represents the app configuration
 type AppConfig struct {
-	Port               int
-	LogLevel           string
-	WriteLogsToConsole bool
-	BypassExtensions   []string
-	ProcessExtensions  []string
-	PreviewBytes       string
-	PreviewEnabled     bool
-	DebuggingHeaders   bool
-	Services           []string
-	ServicesInstances  map[string]*serviceIcapInfo
+	Port                int
+	LogLevel            string
+	WriteLogsToConsole  bool
+	BypassExtensions    []string
+	ProcessExtensions   []string
+	PreviewBytes        string
+	PreviewEnabled      bool
+	DebuggingHeaders    bool
+	Services            []string
+	ServicesInstances   map[string]*serviceIcapInfo
+	MaxFileSizeOnMemory int
+	WritePathOnDisk     string
 }
 
 var AppCfg AppConfig
@@ -48,11 +50,13 @@ func Init() {
 		fmt.Println("app section doesn't exist in config file")
 	}
 	AppCfg = AppConfig{
-		Port:               readValues.ReadValuesInt("app.port"),
-		LogLevel:           readValues.ReadValuesString("app.log_level"),
-		WriteLogsToConsole: readValues.ReadValuesBool("app.write_logs_to_console"),
-		DebuggingHeaders:   readValues.ReadValuesBool("app.debugging_headers"),
-		Services:           readValues.ReadValuesSlice("app.services"),
+		Port:                readValues.ReadValuesInt("app.port"),
+		LogLevel:            readValues.ReadValuesString("app.log_level"),
+		WriteLogsToConsole:  readValues.ReadValuesBool("app.write_logs_to_console"),
+		DebuggingHeaders:    readValues.ReadValuesBool("app.debugging_headers"),
+		Services:            readValues.ReadValuesSlice("app.services"),
+		MaxFileSizeOnMemory: readValues.ReadValuesInt("app.max_file_size_on_memory"),
+		WritePathOnDisk:     readValues.ReadValuesString("app.write_path_on_disk"),
 	}
 	logging.InitializeLogger(AppCfg.LogLevel, AppCfg.WriteLogsToConsole)
 	logging.Logger.Info("Reading config.toml file")

consts/consts.go

@@ -15,7 +15,7 @@ const (
 
 // the common constants
 const (
-	Unknown                           = "unknown"
+	Unknown                           = "!"
 	Any                               = "*"
 	NoModificationStatusCodeStr       = 204
 	BadRequestStatusCodeStr           = 400
@@ -34,7 +34,7 @@ const (
 	ProcessExts                       = "process"
 	RejectExts                        = "reject"
 	BypassExts                        = "bypass"
-	BlockPagePath                     = "block-page.html"
+	BlockPagePath                     = "temp/exception-page.html"
 	ErrPageReasonFileRejected         = "fileRejected"
 	ErrPageReasonMaxFileExceeded      = "maxFileSizeExceeded"
 	ErrPageReasonFileIsNotSafe        = "fileIsNotSafe"

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
+	github.com/google/uuid v1.1.2
 	github.com/h2non/filetype v1.0.12
 	github.com/spf13/viper v1.9.0
 	github.com/xhit/go-str2duration/v2 v2.0.0

go.sum

@@ -152,6 +152,7 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=