');
if any(isTestReport)
idx1 = find(isTestReport);
@@ -126,6 +130,18 @@
characterListPattern("./_-:;%&?#"), 1);
url = extract(content,urlPat);
end
-
+
+ % generate base64 encoded img tag
+ function imgTag = getHTMLImgTag(filepath)
+ [~,~,ext] = fileparts(filepath);
+ MIMEType = "data:image/" + erase(ext,".") + ";base64,";
+ fid = fopen(filepath);
+ byteArray = fread(fid,'*uint8');
+ fclose(fid);
+ b64char = matlab.net.base64encode(byteArray);
+ urlEncoded = MIMEType + b64char;
+ imgTag = "
";
+ end
+
end
end
\ No newline at end of file
diff --git a/helpers/llms-with-matlab/+llms/+internal/callOpenAIChatAPI.m b/helpers/llms-with-matlab/+llms/+internal/callOpenAIChatAPI.m
index 5259fdf..3cd485c 100644
--- a/helpers/llms-with-matlab/+llms/+internal/callOpenAIChatAPI.m
+++ b/helpers/llms-with-matlab/+llms/+internal/callOpenAIChatAPI.m
@@ -84,8 +84,20 @@
if isempty(nvp.StreamFun)
message = response.Body.Data.choices(1).message;
else
- message = struct("role", "assistant", ...
- "content", streamedText);
+ pat = '{"' + wildcardPattern + '":';
+ if contains(streamedText,pat)
+ s = jsondecode(streamedText);
+ if contains(s.function.arguments,pat)
+ prompt = jsondecode(s.function.arguments);
+ s.function.arguments = prompt;
+ end
+ message = struct("role", "assistant", ...
+ "content",[], ...
+ "tool_calls",jsondecode(streamedText));
+ else
+ message = struct("role", "assistant", ...
+ "content", streamedText);
+ end
end
if isfield(message, "tool_choice")
text = "";
@@ -131,7 +143,7 @@
nvpOptions = keys(dict);
if strcmp(nvp.ModelName,'gpt-4-vision-preview')
- nvpOptions(ismember(nvpOptions,["MaxNumTokens","StopSequences"])) = [];
+ nvpOptions(ismember(nvpOptions,"StopSequences")) = [];
end
for opt = nvpOptions.'
diff --git a/helpers/llms-with-matlab/+llms/+stream/responseStreamer.m b/helpers/llms-with-matlab/+llms/+stream/responseStreamer.m
index 58925a6..8db9ff3 100644
--- a/helpers/llms-with-matlab/+llms/+stream/responseStreamer.m
+++ b/helpers/llms-with-matlab/+llms/+stream/responseStreamer.m
@@ -36,14 +36,43 @@
str = erase(str,"data: ");
for i = 1:length(str)
- json = jsondecode(str{i});
- if strcmp(json.choices.finish_reason,'stop')
+ if strcmp(str{i},'[DONE]')
stop = true;
return
else
- txt = json.choices.delta.content;
- this.StreamFun(txt);
- this.ResponseText = [this.ResponseText txt];
+ try
+ json = jsondecode(str{i});
+ catch ME
+ errID = 'llms:stream:responseStreamer:InvalidInput';
+ msg = "Input does not have the expected json format. " + str{i};
+ ME = MException(errID,msg);
+ throw(ME)
+ end
+ if ischar(json.choices.finish_reason) && ismember(json.choices.finish_reason,["stop","tool_calls"])
+ stop = true;
+ return
+ else
+ if isfield(json.choices.delta,"tool_calls")
+ if isfield(json.choices.delta.tool_calls,"id")
+ id = json.choices.delta.tool_calls.id;
+ type = json.choices.delta.tool_calls.type;
+ fcn = json.choices.delta.tool_calls.function;
+ s = struct('id',id,'type',type,'function',fcn);
+ txt = jsonencode(s);
+ else
+ s = jsondecode(this.ResponseText);
+ args = json.choices.delta.tool_calls.function.arguments;
+ s.function.arguments = [s.function.arguments args];
+ txt = jsonencode(s);
+ end
+ this.StreamFun('');
+ this.ResponseText = txt;
+ else
+ txt = json.choices.delta.content;
+ this.StreamFun(txt);
+ this.ResponseText = [this.ResponseText txt];
+ end
+ end
end
end
end
diff --git a/helpers/llms-with-matlab/examples/ExampleEmbeddings.mlx b/helpers/llms-with-matlab/examples/ExampleEmbeddings.mlx
new file mode 100644
index 0000000..c687ce7
Binary files /dev/null and b/helpers/llms-with-matlab/examples/ExampleEmbeddings.mlx differ
diff --git a/helpers/llms-with-matlab/examples/ExampleParallelFunctionCalls.mlx b/helpers/llms-with-matlab/examples/ExampleParallelFunctionCalls.mlx
index 4ea3a46..11ab88c 100644
Binary files a/helpers/llms-with-matlab/examples/ExampleParallelFunctionCalls.mlx and b/helpers/llms-with-matlab/examples/ExampleParallelFunctionCalls.mlx differ
diff --git a/helpers/llms-with-matlab/openAIChat.m b/helpers/llms-with-matlab/openAIChat.m
index fc15fbb..aced1cb 100644
--- a/helpers/llms-with-matlab/openAIChat.m
+++ b/helpers/llms-with-matlab/openAIChat.m
@@ -209,8 +209,8 @@
% reproducible responses
%
% Currently, GPT-4 Turbo with vision does not support the message.name
- % parameter, functions/tools, response_format parameter, stop
- % sequences, and max_tokens
+ % parameter, functions/tools, response_format parameter, and stop
+ % sequences. It also has a low MaxNumTokens default, which can be overridden.
arguments
this (1,1) openAIChat
@@ -221,11 +221,6 @@
nvp.Seed {mustBeIntegerOrEmpty(nvp.Seed)} = []
end
- if nvp.MaxNumTokens ~= Inf && strcmp(this.ModelName,'gpt-4-vision-preview')
- error("llms:invalidOptionForModel", ...
- llms.utils.errorMessageCatalog.getMessage("llms:invalidOptionForModel", "MaxNumTokens", this.ModelName));
- end
-
toolChoice = convertToolChoice(this, nvp.ToolChoice);
if ~isempty(nvp.ToolChoice) && strcmp(this.ModelName,'gpt-4-vision-preview')
error("llms:invalidOptionForModel", ...
@@ -316,8 +311,8 @@ function mustBeValidFunctionCall(this, functionCall)
if ~isempty(this.Tools)
toolChoice = "auto";
end
- elseif ToolChoice ~= "auto"
- % if toolChoice is not empty, then it must be in the format
+ elseif ~ismember(toolChoice,["auto","none"])
+ % if toolChoice is not empty, then it must be "auto", "none" or in the format
% {"type": "function", "function": {"name": "my_function"}}
toolChoice = struct("type","function","function",struct("name",toolChoice));
end