add ollama support
This commit is contained in:
Game_Time
347
chatmock.py
347
chatmock.py
@@ -96,6 +96,62 @@ def create_app(
|
|||||||
reasoning["summary"] = summary
|
reasoning["summary"] = summary
|
||||||
return reasoning
|
return reasoning
|
||||||
|
|
||||||
|
def _to_data_url(image_str: str) -> str:
|
||||||
|
if not isinstance(image_str, str) or not image_str:
|
||||||
|
return image_str
|
||||||
|
s = image_str.strip()
|
||||||
|
if s.startswith("data:image/"):
|
||||||
|
return s
|
||||||
|
if s.startswith("http://") or s.startswith("https://"):
|
||||||
|
return s
|
||||||
|
b64 = s.replace("\n", "").replace("\r", "")
|
||||||
|
kind = "image/png"
|
||||||
|
if b64.startswith("/9j/"):
|
||||||
|
kind = "image/jpeg"
|
||||||
|
elif b64.startswith("iVBORw0KGgo"):
|
||||||
|
kind = "image/png"
|
||||||
|
elif b64.startswith("R0lGOD"):
|
||||||
|
kind = "image/gif"
|
||||||
|
return f"data:{kind};base64,{b64}"
|
||||||
|
|
||||||
|
def _convert_ollama_messages(messages: List[Dict[str, Any]] | None, top_images: List[str] | None) -> List[Dict[str, Any]]:
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
msgs = messages if isinstance(messages, list) else []
|
||||||
|
for m in msgs:
|
||||||
|
if not isinstance(m, dict):
|
||||||
|
continue
|
||||||
|
role = m.get("role") or "user"
|
||||||
|
content = m.get("content")
|
||||||
|
images = m.get("images") if isinstance(m.get("images"), list) else []
|
||||||
|
parts = []
|
||||||
|
if isinstance(content, list):
|
||||||
|
for p in content:
|
||||||
|
if isinstance(p, dict) and p.get("type") == "text" and isinstance(p.get("text"), str):
|
||||||
|
parts.append({"type": "text", "text": p.get("text")})
|
||||||
|
elif isinstance(content, str) and content.strip():
|
||||||
|
parts.append({"type": "text", "text": content})
|
||||||
|
for img in images:
|
||||||
|
url = _to_data_url(img)
|
||||||
|
if isinstance(url, str) and url:
|
||||||
|
parts.append({"type": "image_url", "image_url": {"url": url}})
|
||||||
|
if not parts:
|
||||||
|
parts.append({"type": "text", "text": ""})
|
||||||
|
out.append({"role": role, "content": parts})
|
||||||
|
if isinstance(top_images, list) and top_images:
|
||||||
|
attach_to = None
|
||||||
|
for i in range(len(out) - 1, -1, -1):
|
||||||
|
if out[i].get("role") == "user":
|
||||||
|
attach_to = out[i]
|
||||||
|
break
|
||||||
|
if attach_to is None:
|
||||||
|
attach_to = {"role": "user", "content": []}
|
||||||
|
out.append(attach_to)
|
||||||
|
for img in top_images:
|
||||||
|
url = _to_data_url(img)
|
||||||
|
if isinstance(url, str) and url:
|
||||||
|
attach_to["content"].append({"type": "image_url", "image_url": {"url": url}})
|
||||||
|
return out
|
||||||
|
|
||||||
@app.route("/v1/chat/completions", methods=["POST", "OPTIONS"])
|
@app.route("/v1/chat/completions", methods=["POST", "OPTIONS"])
|
||||||
def chat_completions() -> Response:
|
def chat_completions() -> Response:
|
||||||
if request.method == "OPTIONS":
|
if request.method == "OPTIONS":
|
||||||
@@ -320,6 +376,297 @@ def create_app(
|
|||||||
resp.headers.setdefault(k, v)
|
resp.headers.setdefault(k, v)
|
||||||
return resp
|
return resp
|
||||||
|
|
||||||
|
_OLLAMA_FAKE_EVAL = {
|
||||||
|
"total_duration": 8497226791,
|
||||||
|
"load_duration": 1747193958,
|
||||||
|
"prompt_eval_count": 24,
|
||||||
|
"prompt_eval_duration": 269219750,
|
||||||
|
"eval_count": 247,
|
||||||
|
"eval_duration": 6413802458,
|
||||||
|
}
|
||||||
|
|
||||||
|
@app.route("/api/tags", methods=["GET", "OPTIONS"])
|
||||||
|
def ollama_tags() -> Response:
|
||||||
|
if request.method == "OPTIONS":
|
||||||
|
resp = make_response("", 204)
|
||||||
|
for k, v in build_cors_headers().items():
|
||||||
|
resp.headers[k] = v
|
||||||
|
return resp
|
||||||
|
model_id = "gpt-5"
|
||||||
|
models = [{
|
||||||
|
"name": model_id,
|
||||||
|
"model": model_id,
|
||||||
|
"modified_at": "2023-10-01T00:00:00Z",
|
||||||
|
"size": 815319791,
|
||||||
|
"digest": "8648f39daa8fbf5b18c7b4e6a8fb4990c692751d49917417b8842ca5758e7ffc",
|
||||||
|
"details": {
|
||||||
|
"parent_model": "",
|
||||||
|
"format": "gguf",
|
||||||
|
"family": "llama",
|
||||||
|
"families": ["llama"],
|
||||||
|
"parameter_size": "8.0B",
|
||||||
|
"quantization_level": "Q4_0",
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
resp = make_response(jsonify({"models": models}), 200)
|
||||||
|
for k, v in build_cors_headers().items():
|
||||||
|
resp.headers.setdefault(k, v)
|
||||||
|
return resp
|
||||||
|
|
||||||
|
@app.route("/api/show", methods=["POST", "OPTIONS"])
|
||||||
|
def ollama_show() -> Response:
|
||||||
|
if request.method == "OPTIONS":
|
||||||
|
resp = make_response("", 204)
|
||||||
|
for k, v in build_cors_headers().items():
|
||||||
|
resp.headers[k] = v
|
||||||
|
return resp
|
||||||
|
try:
|
||||||
|
payload = request.get_json(silent=True) or {}
|
||||||
|
except Exception:
|
||||||
|
payload = {}
|
||||||
|
model = payload.get("model")
|
||||||
|
if not isinstance(model, str) or not model.strip():
|
||||||
|
return jsonify({"error": "Model not found"}), 400
|
||||||
|
v1_show_response = {
|
||||||
|
"modelfile": "# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this one, replace the FROM line with:\n# FROM llava:latest\n\nFROM /models/blobs/sha256:placeholder\nTEMPLATE \"\"\"{{ .System }}\nUSER: {{ .Prompt }}\nASSISTANT: \"\"\"\nPARAMETER num_ctx 100000\nPARAMETER stop \"</s>\"\nPARAMETER stop \"USER:\"\nPARAMETER stop \"ASSISTANT:\"",
|
||||||
|
"parameters": "num_keep 24\nstop \"<|start_header_id|>\"\nstop \"<|end_header_id|>\"\nstop \"<|eot_id|>\"",
|
||||||
|
"template": "{{ if .System }}<|start_header_id|>system<|end_header_id|>\n\n{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>\n\n{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>\n\n{{ .Response }}<|eot_id|>",
|
||||||
|
"details": {
|
||||||
|
"parent_model": "",
|
||||||
|
"format": "gguf",
|
||||||
|
"family": "llama",
|
||||||
|
"families": ["llama"],
|
||||||
|
"parameter_size": "8.0B",
|
||||||
|
"quantization_level": "Q4_0",
|
||||||
|
},
|
||||||
|
"model_info": {
|
||||||
|
"general.architecture": "llama",
|
||||||
|
"general.file_type": 2,
|
||||||
|
"llama.context_length": 2000000,
|
||||||
|
},
|
||||||
|
"capabilities": ["completion", "vision"],
|
||||||
|
}
|
||||||
|
resp = make_response(jsonify(v1_show_response), 200)
|
||||||
|
for k, v in build_cors_headers().items():
|
||||||
|
resp.headers.setdefault(k, v)
|
||||||
|
return resp
|
||||||
|
|
||||||
|
@app.route("/api/chat", methods=["POST", "OPTIONS"])
|
||||||
|
def ollama_chat() -> Response:
|
||||||
|
if request.method == "OPTIONS":
|
||||||
|
resp = make_response("", 204)
|
||||||
|
for k, v in build_cors_headers().items():
|
||||||
|
resp.headers[k] = v
|
||||||
|
return resp
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw = request.get_data(cache=True, as_text=True) or ""
|
||||||
|
payload = json.loads(raw) if raw else {}
|
||||||
|
except Exception:
|
||||||
|
return jsonify({"error": "Invalid JSON body"}), 400
|
||||||
|
|
||||||
|
model = payload.get("model")
|
||||||
|
raw_messages = payload.get("messages")
|
||||||
|
messages = _convert_ollama_messages(raw_messages, payload.get("images") if isinstance(payload.get("images"), list) else None)
|
||||||
|
stream_req = payload.get("stream")
|
||||||
|
if stream_req is None:
|
||||||
|
stream_req = True
|
||||||
|
stream_req = bool(stream_req)
|
||||||
|
|
||||||
|
if not isinstance(model, str) or not isinstance(messages, list) or not messages:
|
||||||
|
return jsonify({"error": "Invalid request format"}), 400
|
||||||
|
|
||||||
|
input_items = convert_chat_messages_to_responses_input(messages)
|
||||||
|
|
||||||
|
upstream, error_resp = _start_upstream_request(
|
||||||
|
_normalize_model_name(model),
|
||||||
|
input_items,
|
||||||
|
instructions=BASE_INSTRUCTIONS,
|
||||||
|
tools=[],
|
||||||
|
tool_choice="auto",
|
||||||
|
parallel_tool_calls=False,
|
||||||
|
reasoning_param=_build_reasoning_param(None),
|
||||||
|
)
|
||||||
|
if error_resp is not None:
|
||||||
|
return error_resp
|
||||||
|
|
||||||
|
if upstream.status_code >= 400:
|
||||||
|
try:
|
||||||
|
err_body = json.loads(upstream.content.decode("utf-8", errors="ignore")) if upstream.content else {"raw": upstream.text}
|
||||||
|
except Exception:
|
||||||
|
err_body = {"raw": upstream.text}
|
||||||
|
return (
|
||||||
|
jsonify({"error": (err_body.get("error", {}) or {}).get("message", "Upstream error")}),
|
||||||
|
upstream.status_code,
|
||||||
|
)
|
||||||
|
|
||||||
|
created_at = str(int(time.time() * 1000))
|
||||||
|
|
||||||
|
if stream_req:
|
||||||
|
def _gen():
|
||||||
|
compat = (reasoning_compat or "think-tags").strip().lower()
|
||||||
|
think_open = False
|
||||||
|
think_closed = False
|
||||||
|
saw_any_summary = False
|
||||||
|
pending_summary_paragraph = False
|
||||||
|
try:
|
||||||
|
for raw_line in upstream.iter_lines(decode_unicode=False):
|
||||||
|
if not raw_line:
|
||||||
|
continue
|
||||||
|
line = raw_line.decode("utf-8", errors="ignore") if isinstance(raw_line, (bytes, bytearray)) else raw_line
|
||||||
|
if not line.startswith("data: "):
|
||||||
|
continue
|
||||||
|
data = line[len("data: "):].strip()
|
||||||
|
if not data or data == "[DONE]":
|
||||||
|
if data == "[DONE]":
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
evt = json.loads(data)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
kind = evt.get("type")
|
||||||
|
|
||||||
|
if compat == "think-tags":
|
||||||
|
if kind == "response.reasoning_summary_part.added":
|
||||||
|
if saw_any_summary:
|
||||||
|
pending_summary_paragraph = True
|
||||||
|
else:
|
||||||
|
saw_any_summary = True
|
||||||
|
continue
|
||||||
|
if kind in ("response.reasoning_summary_text.delta", "response.reasoning_text.delta"):
|
||||||
|
delta_txt = evt.get("delta") or ""
|
||||||
|
if not think_open and not think_closed:
|
||||||
|
out = {
|
||||||
|
"model": _normalize_model_name(model),
|
||||||
|
"created_at": created_at,
|
||||||
|
"message": {"role": "assistant", "content": "<think>"},
|
||||||
|
"done": False,
|
||||||
|
}
|
||||||
|
yield json.dumps(out, ensure_ascii=False) + "\n\n"
|
||||||
|
think_open = True
|
||||||
|
if pending_summary_paragraph:
|
||||||
|
out = {
|
||||||
|
"model": _normalize_model_name(model),
|
||||||
|
"created_at": created_at,
|
||||||
|
"message": {"role": "assistant", "content": "\n"},
|
||||||
|
"done": False,
|
||||||
|
}
|
||||||
|
yield json.dumps(out, ensure_ascii=False) + "\n\n"
|
||||||
|
pending_summary_paragraph = False
|
||||||
|
if isinstance(delta_txt, str) and delta_txt:
|
||||||
|
out = {
|
||||||
|
"model": _normalize_model_name(model),
|
||||||
|
"created_at": created_at,
|
||||||
|
"message": {"role": "assistant", "content": delta_txt},
|
||||||
|
"done": False,
|
||||||
|
}
|
||||||
|
yield json.dumps(out, ensure_ascii=False) + "\n\n"
|
||||||
|
continue
|
||||||
|
|
||||||
|
if kind == "response.output_text.delta":
|
||||||
|
if compat == "think-tags" and think_open and not think_closed:
|
||||||
|
outc = {
|
||||||
|
"model": _normalize_model_name(model),
|
||||||
|
"created_at": created_at,
|
||||||
|
"message": {"role": "assistant", "content": "</think>"},
|
||||||
|
"done": False,
|
||||||
|
}
|
||||||
|
yield json.dumps(outc, ensure_ascii=False) + "\n\n"
|
||||||
|
think_open = False
|
||||||
|
think_closed = True
|
||||||
|
chunk = evt.get("delta") or ""
|
||||||
|
if not isinstance(chunk, str) or not chunk:
|
||||||
|
continue
|
||||||
|
out = {
|
||||||
|
"model": _normalize_model_name(model),
|
||||||
|
"created_at": created_at,
|
||||||
|
"message": {"role": "assistant", "content": chunk},
|
||||||
|
"done": False,
|
||||||
|
}
|
||||||
|
yield json.dumps(out, ensure_ascii=False) + "\n\n"
|
||||||
|
elif kind == "response.completed":
|
||||||
|
break
|
||||||
|
finally:
|
||||||
|
if compat == "think-tags" and think_open and not think_closed:
|
||||||
|
outc = {
|
||||||
|
"model": _normalize_model_name(model),
|
||||||
|
"created_at": created_at,
|
||||||
|
"message": {"role": "assistant", "content": "</think>"},
|
||||||
|
"done": False,
|
||||||
|
}
|
||||||
|
yield json.dumps(outc, ensure_ascii=False) + "\n\n"
|
||||||
|
think_open = False
|
||||||
|
think_closed = True
|
||||||
|
upstream.close()
|
||||||
|
final_out = {
|
||||||
|
"model": _normalize_model_name(model),
|
||||||
|
"created_at": created_at,
|
||||||
|
"message": {"role": "assistant", "content": ""},
|
||||||
|
"done": True,
|
||||||
|
"done_reason": "stop",
|
||||||
|
}
|
||||||
|
final_out.update(_OLLAMA_FAKE_EVAL)
|
||||||
|
yield json.dumps(final_out, ensure_ascii=False) + "\n\n"
|
||||||
|
|
||||||
|
resp = Response(_gen(), status=200, mimetype="text/event-stream", headers={"Cache-Control": "no-cache", "Connection": "keep-alive"})
|
||||||
|
for k, v in build_cors_headers().items():
|
||||||
|
resp.headers.setdefault(k, v)
|
||||||
|
return resp
|
||||||
|
|
||||||
|
full_text = ""
|
||||||
|
reasoning_summary_text = ""
|
||||||
|
reasoning_full_text = ""
|
||||||
|
try:
|
||||||
|
for raw_line in upstream.iter_lines(decode_unicode=False):
|
||||||
|
if not raw_line:
|
||||||
|
continue
|
||||||
|
line = raw_line.decode("utf-8", errors="ignore") if isinstance(raw_line, (bytes, bytearray)) else raw_line
|
||||||
|
if not line.startswith("data: "):
|
||||||
|
continue
|
||||||
|
data = line[len("data: "):].strip()
|
||||||
|
if not data or data == "[DONE]":
|
||||||
|
if data == "[DONE]":
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
evt = json.loads(data)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
kind = evt.get("type")
|
||||||
|
if kind == "response.output_text.delta":
|
||||||
|
full_text += evt.get("delta") or ""
|
||||||
|
elif kind == "response.reasoning_summary_text.delta":
|
||||||
|
reasoning_summary_text += evt.get("delta") or ""
|
||||||
|
elif kind == "response.reasoning_text.delta":
|
||||||
|
reasoning_full_text += evt.get("delta") or ""
|
||||||
|
elif kind == "response.completed":
|
||||||
|
break
|
||||||
|
finally:
|
||||||
|
upstream.close()
|
||||||
|
compat = (reasoning_compat or "think-tags").strip().lower()
|
||||||
|
if compat == "think-tags":
|
||||||
|
rtxt_parts = []
|
||||||
|
if isinstance(reasoning_summary_text, str) and reasoning_summary_text.strip():
|
||||||
|
rtxt_parts.append(reasoning_summary_text)
|
||||||
|
if isinstance(reasoning_full_text, str) and reasoning_full_text.strip():
|
||||||
|
rtxt_parts.append(reasoning_full_text)
|
||||||
|
rtxt = "\n\n".join([p for p in rtxt_parts if p])
|
||||||
|
if rtxt:
|
||||||
|
full_text = f"<think>{rtxt}</think>" + (full_text or "")
|
||||||
|
out_json = {
|
||||||
|
"model": _normalize_model_name(model),
|
||||||
|
"created_at": created_at,
|
||||||
|
"message": {"role": "assistant", "content": full_text},
|
||||||
|
"done": True,
|
||||||
|
"done_reason": "stop",
|
||||||
|
}
|
||||||
|
out_json.update(_OLLAMA_FAKE_EVAL)
|
||||||
|
resp = make_response(jsonify(out_json), 200)
|
||||||
|
for k, v in build_cors_headers().items():
|
||||||
|
resp.headers.setdefault(k, v)
|
||||||
|
return resp
|
||||||
|
|
||||||
@app.route("/v1/models", methods=["GET", "OPTIONS"])
|
@app.route("/v1/models", methods=["GET", "OPTIONS"])
|
||||||
def list_models() -> Response:
|
def list_models() -> Response:
|
||||||
if request.method == "OPTIONS":
|
if request.method == "OPTIONS":
|
||||||
|
|||||||
Reference in New Issue
Block a user