geoffsee/open-gsio
1
0
Fork 0
mirror of https://github.com/geoffsee/open-gsio.git synced 2025-09-08 22:56:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixes model initialization for mlx

Browse Source
This commit is contained in:
geoffsee
2025-06-18 13:30:38 -04:00
parent 38b364caeb
commit f1d7f52dbd
12 changed files with 127 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/client/src/pages/index/+Page.tsx
View File

@@ -2,7 +2,6 @@ import React, { useEffect } from "react";
import { Stack } from "@chakra-ui/react";
import Chat from "../../components/chat/Chat";
import clientChatStore from "../../stores/ClientChatStore";
import { getModelFamily } from "../../components/chat/lib/SupportedModels";
// renders "/"
export default function IndexPage() {

10
packages/client/src/stores/UserOptionsStore.ts
View File

@@ -23,7 +23,7 @@ export const UserOptionsStoreModel = types
.split(";")
.find((row) => row.startsWith("user_preferences"));
console.log(document.cookie.split(";"));
// console.log(document.cookie.split(";"));
const newUserOptions = JSON.stringify({
theme: self.theme,
@@ -46,7 +46,7 @@ export const UserOptionsStoreModel = types
.find((row) => row.startsWith("user_preferences"));
if (!userPreferencesCoookie) {
console.log("No user preferences cookie found, creating one");
// console.log("No user preferences cookie found, creating one");
self.storeUserOptions();
}
@@ -60,20 +60,20 @@ export const UserOptionsStoreModel = types
window.addEventListener("scroll", () => {
if (ClientChatStore.isLoading && self.followModeEnabled) {
console.log("scrolling");
// console.log("scrolling");
self.setFollowModeEnabled(false);
}
});
window.addEventListener("wheel", () => {
if (ClientChatStore.isLoading && self.followModeEnabled) {
console.log("wheel");
// console.log("wheel");
self.setFollowModeEnabled(false);
}
});
window.addEventListener("touchmove", () => {
console.log("touchmove");
// console.log("touchmove");
if (ClientChatStore.isLoading && self.followModeEnabled) {
self.setFollowModeEnabled(false);
}

4
packages/env/env.d.ts vendored
View File

@@ -12,7 +12,9 @@ interface Env {
// KV Bindings
KV_STORAGE: KVNamespace;
// Text/Secrets
METRICS_HOST: string;
OPENAI_API_ENDPOINT: string;
OPENAI_API_KEY: string;
EVENTSOURCE_HOST: string;
@@ -24,4 +26,6 @@ interface Env {
CEREBRAS_API_KEY: string;
CLOUDFLARE_API_KEY: string;
CLOUDFLARE_ACCOUNT_ID: string;
MLX_API_KEY: string;
OLLAMA_API_KEY: string;
}

7
packages/scripts/configure_local_inference.sh
View File

@@ -17,11 +17,18 @@ configure_dev_vars() {
# Default URL is automatic but can be overridden for remote deployments
if [[ "$endpoint_url" == *"11434"* ]]; then
echo "OPENAI_API_ENDPOINT=http://localhost:11434" >> "${ENV_LOCAL_PATH}"
echo "OLLAMA_API_KEY=active" >> "${ENV_LOCAL_PATH}"
echo "OPENAI_API_ENDPOINT=http://localhost:11434" >> "${DEV_VARS_PATH}"
echo "OLLAMA_API_KEY=active" >> "${DEV_VARS_PATH}"
fi
if [[ "$endpoint_url" == *"10240"* ]]; then
echo "OPENAI_API_ENDPOINT=http://localhost:10240/v1" >> "${ENV_LOCAL_PATH}"
echo "MLX_API_KEY=active" >> "${ENV_LOCAL_PATH}"
echo "OPENAI_API_ENDPOINT=http://localhost:10240/v1" >> "${DEV_VARS_PATH}"
echo "MLX_API_KEY=active" >> "${DEV_VARS_PATH}"
fi

7
packages/server/api-router.ts
View File

@@ -57,14 +57,15 @@ export function createRouter() {
// return documentService.handleGetDocument(r)
// })
.all("/api/metrics/*", async (r, e, c) => {
.all("/api/metrics*", async (r, e, c) => {
const { metricsService } = createRequestContext(e, c);
return metricsService.handleMetricsRequest(r);
})
// renders the app
.get('*', async (r, e, c) => {
const { assetService } = createRequestContext(e, c);
.get("^(?!/api/).*$", async (r, e, c) => {
const { assetService } = createRequestContext(e, c);
console.log('Request received:', { url: r.url, headers: r.headers });

6
packages/server/lib/assistant-sdk.ts
View File

@@ -13,13 +13,13 @@ export class AssistantSdk {
userLocation = "",
} = params;
// Handle both nested and flat few_shots structures
console.log('[DEBUG_LOG] few_shots:', JSON.stringify(few_shots));
// console.log('[DEBUG_LOG] few_shots:', JSON.stringify(few_shots));
let selectedFewshots = Utils.selectEquitably?.(few_shots);
console.log('[DEBUG_LOG] selectedFewshots after Utils.selectEquitably:', JSON.stringify(selectedFewshots));
// console.log('[DEBUG_LOG] selectedFewshots after Utils.selectEquitably:', JSON.stringify(selectedFewshots));
if (!selectedFewshots) {
// If Utils.selectEquitably returns undefined, use few_shots directly
selectedFewshots = few_shots;
console.log('[DEBUG_LOG] selectedFewshots after fallback:', JSON.stringify(selectedFewshots));
// console.log('[DEBUG_LOG] selectedFewshots after fallback:', JSON.stringify(selectedFewshots));
}
const sdkDate = new Date().toISOString();
const [currentDate] = sdkDate.includes("T") ? sdkDate.split("T") : [sdkDate];

4
packages/server/lib/chat-sdk.ts
View File

@@ -35,8 +35,8 @@ export class ChatSdk {
const preprocessedContext = await ChatSdk.preprocess({
messages,
});
console.log(ctx.env)
console.log(ctx.env.SERVER_COORDINATOR);
// console.log(ctx.env)
// console.log(ctx.env.SERVER_COORDINATOR);
const objectId = ctx.env.SERVER_COORDINATOR.idFromName("stream-index");
const durableObject = ctx.env.SERVER_COORDINATOR.get(objectId);

64
packages/server/providers/_ProviderRepository.ts
View File

@@ -16,7 +16,7 @@ export class ProviderRepository {
openai: 'https://api.openai.com/v1/',
cerebras: 'https://api.cerebras.com/v1/',
ollama: "http://localhost:11434",
mlx: "http://localhost:10240",
mlx: "http://localhost:10240/v1",
}
static async getModelFamily(model, env: Env) {
@@ -42,33 +42,41 @@ export class ProviderRepository {
for (let i = 0; i < envKeys.length; i++) {
if (envKeys[i].endsWith('KEY')) {
const detectedProvider = envKeys[i].split('_')[0].toLowerCase();
switch (detectedProvider) {
case 'anthropic':
this.#providers.push({
name: 'anthropic',
key: env.ANTHROPIC_API_KEY,
endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['anthropic']
});
break;
case 'gemini':
this.#providers.push({
name: 'google',
key: env.GEMINI_API_KEY,
endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['google']
});
break;
case 'cloudflare':
this.#providers.push({
name: 'cloudflare',
key: env.CLOUDFLARE_API_KEY,
endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider].replace("{CLOUDFLARE_ACCOUNT_ID}", env.CLOUDFLARE_ACCOUNT_ID)
})
default:
this.#providers.push({
name: detectedProvider,
key: env[envKeys[i]],
endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider]
});
const detectedProviderValue = env[envKeys[i]];
if(detectedProviderValue) {
console.log({detectedProviderValue});
switch (detectedProvider) {
case 'anthropic':
console.log({detectedProvider});
this.#providers.push({
name: 'anthropic',
key: env.ANTHROPIC_API_KEY,
endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['anthropic']
});
break;
case 'gemini':
console.log({detectedProvider});
this.#providers.push({
name: 'google',
key: env.GEMINI_API_KEY,
endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['google']
});
break;
case 'cloudflare':
console.log({detectedProvider});
this.#providers.push({
name: 'cloudflare',
key: env.CLOUDFLARE_API_KEY,
endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider].replace("{CLOUDFLARE_ACCOUNT_ID}", env.CLOUDFLARE_ACCOUNT_ID)
})
default:
console.log({detectedProvider});
this.#providers.push({
name: detectedProvider,
key: env[envKeys[i]],
endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider]
});
}
}
}
}

34
packages/server/server.ts
View File

@@ -8,24 +8,32 @@ import DurableObjectLocal from "./ServerCoordinatorBun";
const router = Server.Router();
config({ path: ['./.env'] })
config({
path: ".env",
debug: true,
// defaults: {
// EVENTSOURCE_HOST: "https://eventsource.seemueller.io",
// }
})
export default {
port: 3003,
fetch: async (request: RequestLike, env: { [key: string]: any; }, ctx: any) =>{
console.log("[trace] request: ", request.method, request.url, "headers: ", request.headers.get("referer"), "body: ", request.body, "env: ", env, "ctx: ", ctx, "")
// console.log("[trace] request: ", request.method, request.url, "headers: ", request.headers.get("referer"), "body: ", request.body, "env: ", env, "ctx: ", ctx, "")
env["SERVER_COORDINATOR"] = DurableObjectLocal
env["ASSETS"] = assetHandler.ASSETS
env["EVENTSOURCE_HOST"] = process.env.EVENTSOURCE_HOST
env["GROQ_API_KEY"] = process.env.GROQ_API_KEY
env["ANTHROPIC_API_KEY"] = process.env.ANTHROPIC_API_KEY
env["FIREWORKS_API_KEY"] = process.env.FIREWORKS_API_KEY
env["XAI_API_KEY"] = process.env.XAI_API_KEY
env["CEREBRAS_API_KEY"] = process.env.CEREBRAS_API_KEY
env["CLOUDFLARE_API_KEY"] = process.env.CLOUDFLARE_API_KEY
env["CLOUDFLARE_ACCOUNT_ID"] = process.env.CLOUDFLARE_ACCOUNT_ID
env["KV_STORAGE"] = new BunSqliteKVNamespace({namespace: "open-gsio"})
env["SERVER_COORDINATOR"] = DurableObjectLocal;
env["ASSETS"] = assetHandler.ASSETS;
env["EVENTSOURCE_HOST"] = process.env.EVENTSOURCE_HOST;
env["GROQ_API_KEY"] = process.env.GROQ_API_KEY;
env["ANTHROPIC_API_KEY"] = process.env.ANTHROPIC_API_KEY;
env["FIREWORKS_API_KEY"] = process.env.FIREWORKS_API_KEY;
env["XAI_API_KEY"] = process.env.XAI_API_KEY;
env["CEREBRAS_API_KEY"] = process.env.CEREBRAS_API_KEY;
env["CLOUDFLARE_API_KEY"] = process.env.CLOUDFLARE_API_KEY;
env["CLOUDFLARE_ACCOUNT_ID"] = process.env.CLOUDFLARE_ACCOUNT_ID;
env["MLX_API_KEY"] = process.env.MLX_API_KEY;
env["OLLAMA_API_KEY"] = process.env.OLLAMA_API_KEY;
env["KV_STORAGE"] = new BunSqliteKVNamespace({namespace: "open-gsio"});
try {

31
packages/server/services/ChatService.ts
View File

@@ -126,31 +126,38 @@ const ChatService = types
// ----- Helpers ----------------------------------------------------------
const logger = console;
// ----- 1. Try cached value ---------------------------------------------
try {
const cached = yield self.env.KV_STORAGE.get('supportedModels');
if (cached) {
const parsed = JSON.parse(cached as string);
if (Array.isArray(parsed)) {
logger.info('Cache hit – returning supportedModels from KV');
return new Response(JSON.stringify(parsed), { status: 200 });
const useCache = false;
if(useCache) {
// ----- 1. Try cached value ---------------------------------------------
try {
const cached = yield self.env.KV_STORAGE.get('supportedModels');
if (cached) {
const parsed = JSON.parse(cached as string);
if (Array.isArray(parsed) && parsed.length > 0) {
logger.info('Cache hit – returning supportedModels from KV');
return new Response(JSON.stringify(parsed), { status: 200 });
}
logger.warn('Cache entry malformed – refreshing');
}
logger.warn('Cache entry malformed – refreshing');
} catch (err) {
logger.error('Error reading/parsing supportedModels cache', err);
}
} catch (err) {
logger.error('Error reading/parsing supportedModels cache', err);
}
// ----- 2. Build fresh list ---------------------------------------------
const providerRepo = new ProviderRepository(self.env);
const providers = providerRepo.getProviders();
console.log({ providers })
const providerModels = new Map<string, any[]>();
const modelMeta = new Map<string, any>();
for (const provider of providers) {
if (!provider.key) continue;
logger.info(`Fetching models for provider «${provider.name}»`);
logger.info(`Fetching models from «${provider.endpoint}»`);
const openai = new OpenAI({ apiKey: provider.key, baseURL: provider.endpoint });

32
packages/server/services/MetricsService.ts
View File

@@ -17,20 +17,32 @@ const MetricsService = types
},
handleMetricsRequest: flow(function* (request: Request) {
const url = new URL(request.url);
const proxyUrl = `https://metrics.seemueller.io${url.pathname}${url.search}`;
let proxyUrl = "";
if(self.env.METRICS_HOST) {
proxyUrl = new URL(`${self.env.METRICS_HOST}${url.pathname}${url.search}`).toString();
}
try {
const response = yield fetch(proxyUrl, {
if(proxyUrl) {
try {
const response = yield fetch(proxyUrl, {
method: request.method,
headers: request.headers,
body: ["GET", "HEAD"].includes(request.method) ? null : request.body,
redirect: "follow",
});
return response;
} catch (error) {
console.error("Failed to proxy metrics request:", error);
return new Response("metrics misconfigured", { status: 200 });
}
} else {
const event = {
method: request.method,
headers: request.headers,
body: ["GET", "HEAD"].includes(request.method) ? null : request.body,
redirect: "follow",
});
return response;
} catch (error) {
console.error("Failed to proxy metrics request:", error);
return new Response("Failed to fetch metrics", { status: 500 });
}
self.env.KV_STORAGE.put(`metrics_events::${crypto.randomUUID()}`, JSON.stringify(event));
}
}),
}));

8
packages/server/services/TransactionService.ts
View File

@@ -39,7 +39,7 @@ const TransactionService = types
`https://wallets.seemueller.io${CreateWalletEndpoints[currency]}`,
);
const walletResponse = await walletRequest.text();
console.log({ walletRequest: walletResponse });
// console.log({ walletRequest: walletResponse });
const [address, privateKey, publicKey, phrase] =
JSON.parse(walletResponse);
@@ -56,12 +56,12 @@ const TransactionService = types
phrase,
};
console.log({ txRecord });
// console.log({ txRecord });
const key = `transactions::prepared::${txKey}`;
await self.env.KV_STORAGE.put(key, JSON.stringify(txRecord));
console.log(`PREPARED TRANSACTION ${key}`);
// console.log(`PREPARED TRANSACTION ${key}`);
return {
depositAddress: address,
@@ -72,7 +72,7 @@ const TransactionService = types
handleTransact: async function (request: Request) {
try {
const raw = await request.text();
console.log({ raw });
// console.log({ raw });
const [action, ...payload] = raw.split(",");
const response = await self.routeAction(action, payload);