diff --git a/README.md b/README.md index f5a5463..4169269 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,9 @@ Copyright 2024 Google LLC **Human I/O** is a research project that aims to propose a unified approach to detecting a wide range of Situationally Induced Impairments and Disabilities -(SIIDs) by predicting the availability of human input/output channels. Leveraging -egocentric vision, multimodal sensing and reasoning with large language models. +(SIIDs) by predicting the availability of human input/output channels. +Leveraging egocentric vision, multimodal sensing and reasoning with large +language models. Please cite Human I/O as follows if you find it useful in your projects: @@ -40,17 +41,26 @@ is [Sensible Agent](https://duruofei.com/projects/sensibleai/): } ``` -We encourage future contributors to [XR Blocks](https://github.com/google/xrblocks) for a cross-platform, open-source framework for AI + XR innovations. Example demos of Gemini in XR Blocks include: +We encourage future contributors to +[XR Blocks](https://github.com/google/xrblocks) for a cross-platform, +open-source framework for AI + XR innovations. Example demos of Gemini in XR +Blocks include: -* Gemini Icebreakers: https://xrblocks.github.io/docs/samples/Gemini-Icebreakers/ -* XR Poet: https://xrblocks.github.io/docs/samples/XR-Poet/ -* XR Object: https://xrblocks.github.io/docs/samples/Gemini-XRObject/ +* Gemini Icebreakers: + https://xrblocks.github.io/docs/samples/Gemini-Icebreakers/ +* XR Poet: https://xrblocks.github.io/docs/samples/XR-Poet/ +* XR Object: https://xrblocks.github.io/docs/samples/Gemini-XRObject/ ## API Keys -To run Human I/O, you need to provide your own API keys in `server.js` and `util.js` files for the web app to work properly. Search for `YOUR API KEY` +To run Human I/O, set the `OPENAI_API_KEY` environment variable before starting +the server (e.g. `export OPENAI_API_KEY=sk-...`). The browser never sees the +key; all OpenAI calls are proxied through the `/openaiProxy` route in +`server.js`. -In addition, you need to place your [Google Cloud Vision AI](https://cloud.google.com/vision) credentials (`key.pem` and `cert.pem`) in the folder. +In addition, you need to place your +[Google Cloud Vision AI](https://cloud.google.com/vision) credentials (`key.pem` +and `cert.pem`) in the folder. ## Getting Started @@ -64,17 +74,27 @@ npm start ### View localhost web app on mobile devices -Download and install `ngrok` on your computer. -Start `ngrok` and expose the port that your web app is running on. For example, if your web app is running on port 8000, type `ngrok http 8000` in the Terminal. -Open the browser on your Android phone and type in the URL that `ngrok` provides. For example, . +Download and install `ngrok` on your computer. Start `ngrok` and expose the port +that your web app is running on. For example, if your web app is running on port +8000, type `ngrok http 8000` in the Terminal. Open the browser on your Android +phone and type in the URL that `ngrok` provides. For example, +. ## Using Human I/O -You can use Human I/O, you can either use it with your webcam. You can select the webcam device from the `cameraSelect` dropdown. +You can use Human I/O, you can either use it with your webcam. You can select +the webcam device from the `cameraSelect` dropdown. -In addition, you can run Human I/O on local videos. Put your video (.mp4 format) into the video folder, enter the video filename (without .mp4), and click `load`. +In addition, you can run Human I/O on local videos. Put your video (.mp4 format) +into the video folder, enter the video filename (without .mp4), and click +`load`. ## Blog Post -[Human I/O: Detecting situational impairments with large language models -](https://research.google/blog/human-io-detecting-situational-impairments-with-large-language-models/) +[Human I/O: Detecting situational impairments with large language models](https://research.google/blog/human-io-detecting-situational-impairments-with-large-language-models/) + +## Notice + +This is not an officially supported Google product. This project is not eligible +for the +[Google Open Source Software Vulnerability Rewards Program](https://bughunters.google.com/open-source-security). diff --git a/webapp/server.js b/webapp/server.js index c29a956..6ce2b0e 100644 --- a/webapp/server.js +++ b/webapp/server.js @@ -44,6 +44,32 @@ httpsServer.listen(3000, () => { console.log('HTTPS server running on https://localhost:3000'); }); +// Proxies OpenAI API calls so the API key never reaches the browser. +const OPENAI_ALLOWED_ENDPOINTS = + /^(chat\/completions|completions|engines\/[A-Za-z0-9._-]+\/completions)$/; +app.post('/openaiProxy', async (req, res) => { + const apiKey = process.env.OPENAI_API_KEY || ''; + const endpoint = req.body && req.body.endpoint; + const payload = req.body && req.body.payload; + if (typeof endpoint !== 'string' || !OPENAI_ALLOWED_ENDPOINTS.test(endpoint)) { + return res.status(400).json({error: 'invalid endpoint'}); + } + try { + const upstream = await fetch('https://api.openai.com/v1/' + endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}` + }, + body: JSON.stringify(payload || {}) + }); + const data = await upstream.json(); + res.status(upstream.status).json(data); + } catch (err) { + res.status(502).json({error: 'upstream request failed'}); + } +}); + // Post function to get image caption. app.post('/imageCaption', async (req, res) => { const image = req.body.image; diff --git a/webapp/util.js b/webapp/util.js index 34a84f1..c149d0d 100644 --- a/webapp/util.js +++ b/webapp/util.js @@ -34,26 +34,26 @@ async function callGPT(prompt) { } async function completeChat(prompt) { - const apiKey = "YOUR API KEY"; - const endpoint = "https://api.openai.com/v1/chat/completions"; // const model = "gpt-3.5-turbo"; const model = "gpt-4o"; const messages = [{"role": "user", "content": prompt}]; const temperature = 0.0; - - const response = await fetch(endpoint, { + + const response = await fetch('/openaiProxy', { method: "POST", headers: { - "Content-Type": "application/json", - "Authorization": `Bearer ${apiKey}` + "Content-Type": "application/json" }, body: JSON.stringify({ - model, - messages, - temperature + endpoint: "chat/completions", + payload: { + model, + messages, + temperature + } }) }); - + const data = await response.json(); return data; } @@ -71,7 +71,6 @@ async function callGPTLite(prompt) { } async function completeChatcallGPTLite(prompt) { - const apiKey = "YOUR API KEY"; const data = { "prompt": prompt, "max_tokens": 64, "temperature": 0.0, @@ -82,15 +81,15 @@ async function completeChatcallGPTLite(prompt) { let model_curie = "text-curie-001"; let model_babbage = "babbage-002"; let model_ada = "text-ada-001"; - const response = await fetch( - "https://api.openai.com/v1/engines/" + model_babbage + "/completions", - { + const response = await fetch('/openaiProxy', { headers: { - "Content-Type": 'application/json', - Authorization: "Bearer " + apiKey + "Content-Type": 'application/json' }, method: "POST", - body: JSON.stringify(data), + body: JSON.stringify({ + endpoint: "engines/" + model_babbage + "/completions", + payload: data + }), } ); const result = await response.json();